From 93a4d29c032bec4726372e9db4a1ffff96462fac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81ngel?= Date: Tue, 17 Nov 2026 13:26:15 +0000 Subject: [PATCH] fix(studio): audio meters and audio groups work in a sealed preview (#8011) * test(studio): audio meters and group volume in a sealed preview * fix(studio): audio meters and audio groups work in a sealed preview * chore(inspector): raise the unpacked size budget for the preview's audio editing kinds * test(studio): the sealed audio test serves a Studio-marked preview, as the editing port now requires --- .../src/runtime/editing/assignAudioGroup.ts | 76 ++++++ .../src/runtime/editing/audioKinds.test.ts | 154 +++++++++++ .../core/src/runtime/editing/audioMeter.ts | 28 ++ .../src/runtime/editing/documentMarkup.ts | 3 + .../runtime/editing/setAudioGroupAttribute.ts | 49 ++++ .../core/src/runtime/editingAgent.test.ts | 4 + packages/core/src/runtime/editingAgent.ts | 8 + packages/core/src/runtime/editingPort.test.ts | 4 + packages/core/src/runtime/editingPort.ts | 24 ++ .../src/components/nle/AudioMeterStrip.tsx | 65 ++++- .../components/nle/audioSealedPreview.test.ts | 254 ++++++++++++++++++ .../src/hooks/timelineAudioGroupCreate.ts | 64 ++++- .../src/hooks/timelineAudioGroupVolume.ts | 144 +++++++--- .../player/components/TimelineTrackHeader.tsx | 16 +- .../player/components/clipMenuAudioItems.tsx | 49 +++- .../studio/src/player/lib/previewChannel.ts | 10 + scripts/package-size-budgets.json | 2 +- 17 files changed, 903 insertions(+), 51 deletions(-) create mode 100644 packages/core/src/runtime/editing/assignAudioGroup.ts create mode 100644 packages/core/src/runtime/editing/audioKinds.test.ts create mode 100644 packages/core/src/runtime/editing/audioMeter.ts create mode 100644 packages/core/src/runtime/editing/documentMarkup.ts create mode 100644 packages/core/src/runtime/editing/setAudioGroupAttribute.ts create mode 100644 packages/studio/src/components/nle/audioSealedPreview.test.ts diff --git a/packages/core/src/runtime/editing/assignAudioGroup.ts b/packages/core/src/runtime/editing/assignAudioGroup.ts new file mode 100644 index 0000000000..37f3ddb942 --- /dev/null +++ b/packages/core/src/runtime/editing/assignAudioGroup.ts @@ -0,0 +1,76 @@ +// `assign-audio-group`: a live regroup as Studio's group create and its unwind write it: each member's +// `data-audio-group`, then the group's own `` added or removed. Writes no file. +import { HF_AUDIO_GROUP_ATTR, HF_AUDIO_GROUP_TAG } from "../../audioGroups"; +import { EDITING_ELEMENT_HANDLE, MAX_DESCRIBE_HANDLES, MAX_ELEMENT_VALUE } from "../editingElement"; +import type { EditingElementHandle } from "../editingElement"; +import { audioGroupElement, isGroupId } from "./setAudioGroupAttribute"; + +type Member = { handle: EditingElementHandle; group: string | null }; +type GroupElement = { id: string; label: string | null; present: boolean }; +export type AssignAudioGroupParams = { members: Member[]; element: GroupElement | null }; + +const isRecord = (value: unknown): value is Record => + Object.prototype.toString.call(value) === "[object Object]"; +const keysAre = (value: Record, keys: string) => + Object.keys(value).sort().join() === keys; + +function member(value: unknown): Member | null { + if (!isRecord(value) || !keysAre(value, "group,handle")) return null; + const { handle, group } = value; + const valid = + typeof handle === "string" && + EDITING_ELEMENT_HANDLE.test(handle) && + (group === null || isGroupId(group)); + return valid ? { handle, group: group as string | null } : null; +} + +function groupElement(value: unknown): GroupElement | null { + if (!isRecord(value) || !keysAre(value, "id,label,present")) return null; + const { id, label, present } = value; + const labelOk = + label === null || (typeof label === "string" && label.length <= MAX_ELEMENT_VALUE); + return isGroupId(id) && labelOk && typeof present === "boolean" + ? { id, label: label as string | null, present } + : null; +} + +export function parseAssignAudioGroup(params: unknown): AssignAudioGroupParams | null { + if (!isRecord(params) || !keysAre(params, "element,members")) return null; + const { members, element } = params; + if (!Array.isArray(members) || members.length > MAX_DESCRIBE_HANDLES) return null; + const parsed = members.map(member); + const group = element === null ? null : groupElement(element); + if (parsed.includes(null) || (element !== null && !group)) return null; + return { members: parsed as Member[], element: group }; +} + +/** Adds the group element when absent, or removes it when it is one; true when either happened. */ +function placeGroupElement({ id, label, present }: GroupElement): boolean { + if (!present) { + const existing = audioGroupElement(id); + existing?.remove(); + return existing !== null; + } + if (!document.body || document.getElementById(id)) return false; + const el = document.createElement(HF_AUDIO_GROUP_TAG); + el.id = id; + if (label) el.setAttribute("data-label", label); + document.body.appendChild(el); + return true; +} + +/** `previous` is each member's group before this write, null for none or a member that is gone. */ +export function assignAudioGroup( + { members, element }: AssignAudioGroupParams, + elementForHandle: (handle: EditingElementHandle) => Element | null, +): { previous: Array; changed: boolean } { + const previous = members.map(({ handle, group }) => { + const el = elementForHandle(handle); + if (!el) return null; + const prior = el.getAttribute(HF_AUDIO_GROUP_ATTR); + if (group) el.setAttribute(HF_AUDIO_GROUP_ATTR, group); + else el.removeAttribute(HF_AUDIO_GROUP_ATTR); + return prior; + }); + return { previous, changed: element ? placeGroupElement(element) : false }; +} diff --git a/packages/core/src/runtime/editing/audioKinds.test.ts b/packages/core/src/runtime/editing/audioKinds.test.ts new file mode 100644 index 0000000000..a782f68a97 --- /dev/null +++ b/packages/core/src/runtime/editing/audioKinds.test.ts @@ -0,0 +1,154 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { MAX_DESCRIBE_HANDLES } from "../editingElement"; +import { assignAudioGroup, parseAssignAudioGroup } from "./assignAudioGroup"; +import { audioMeter, parseAudioMeter } from "./audioMeter"; +import { documentMarkup } from "./documentMarkup"; +import { + MAX_GROUP_VALUE, + parseSetAudioGroupAttribute, + setAudioGroupAttribute, +} from "./setAudioGroupAttribute"; + +const H1 = "0123456789abcdef:1"; +const H2 = "0123456789abcdef:2"; + +afterEach(() => { + document.body.innerHTML = ""; +}); + +describe("audio-meter", () => { + it("takes only an on flag", () => { + expect(parseAudioMeter({ on: true })).toEqual({ on: true }); + expect(parseAudioMeter({ on: "yes" })).toBeNull(); + expect(parseAudioMeter({ on: false, extra: 1 })).toBeNull(); + expect(parseAudioMeter([])).toBeNull(); + }); + + it("starts and reads the taps, stops them, and reads nothing without an audio graph", () => { + expect(audioMeter({ on: true }, undefined)).toEqual({ levels: null }); + const levels = { master: { l: 0.5, r: 0.25 }, groups: { vo: { l: 1, r: 1 } } }; + const meter = { start: vi.fn(), stop: vi.fn(), read: vi.fn(() => levels) }; + expect(audioMeter({ on: true }, meter)).toEqual({ levels }); + expect(meter.start).toHaveBeenCalledOnce(); + expect(audioMeter({ on: false }, meter)).toEqual({ levels: null }); + expect(meter.stop).toHaveBeenCalledOnce(); + }); +}); + +describe("document-markup", () => { + it("returns the live markup and base URL", () => { + document.body.innerHTML = ``; + const { html, baseURI } = documentMarkup(); + expect(html).toContain(''); + expect(baseURI).toBe(document.baseURI); + }); +}); + +describe("set-audio-group-attribute", () => { + it("takes a group id, one of the group's own attributes, and a bounded value", () => { + const valid = { id: "vo", attr: "data-volume", value: "0.5" }; + expect(parseSetAudioGroupAttribute(valid)).toEqual(valid); + expect(parseSetAudioGroupAttribute({ ...valid, value: null })).toEqual({ + ...valid, + value: null, + }); + for (const bad of [ + { ...valid, attr: "onclick" }, + { ...valid, attr: "data-audio-group" }, + { ...valid, id: "" }, + { ...valid, value: "x".repeat(MAX_GROUP_VALUE + 1) }, + { ...valid, value: 1 }, + { id: "vo", attr: "data-volume" }, + ]) { + expect(parseSetAudioGroupAttribute(bad)).toBeNull(); + } + }); + + it("writes and removes the attribute on the group element only", () => { + document.body.innerHTML = `
`; + const group = document.getElementById("vo")!; + expect(setAudioGroupAttribute({ id: "vo", attr: "data-volume", value: "2" })).toEqual({ + applied: true, + }); + expect(group.getAttribute("data-volume")).toBe("2"); + setAudioGroupAttribute({ id: "vo", attr: "data-volume", value: null }); + expect(group.hasAttribute("data-volume")).toBe(false); + expect(setAudioGroupAttribute({ id: "card", attr: "data-hidden", value: "" })).toEqual({ + applied: false, + }); + expect(document.getElementById("card")!.hasAttribute("data-hidden")).toBe(false); + }); +}); + +describe("assign-audio-group", () => { + it("takes handle-addressed members and an optional group element", () => { + const valid = { + members: [{ handle: H1, group: "vo" }], + element: { id: "vo", label: null, present: true }, + }; + expect(parseAssignAudioGroup(valid)).toEqual(valid); + expect(parseAssignAudioGroup({ ...valid, element: null })).toEqual({ ...valid, element: null }); + const many = Array.from({ length: MAX_DESCRIBE_HANDLES + 1 }, () => ({ + handle: H1, + group: null, + })); + for (const bad of [ + { ...valid, members: [{ handle: "card", group: "vo" }] }, + { ...valid, members: [{ handle: H1, group: "" }] }, + { ...valid, members: many }, + { ...valid, element: { id: "vo", present: true } }, + { ...valid, element: { id: "vo", label: null, present: "yes" } }, + { members: valid.members }, + ]) { + expect(parseAssignAudioGroup(bad)).toBeNull(); + } + }); + + it("groups the members, adds the group element, and unwinds both", () => { + document.body.innerHTML = ``; + const byHandle = new Map([ + [H1, document.getElementById("a")], + [H2, document.getElementById("b")], + ]); + const resolve = (handle: string) => byHandle.get(handle) ?? null; + const done = assignAudioGroup( + { + members: [ + { handle: H1, group: "vo" }, + { handle: H2, group: "vo" }, + ], + element: { id: "vo", label: "Voice", present: true }, + }, + resolve, + ); + expect(done).toEqual({ previous: ["old", null], changed: true }); + expect(byHandle.get(H1)!.getAttribute("data-audio-group")).toBe("vo"); + expect(document.querySelector("hf-audio-group#vo")!.getAttribute("data-label")).toBe("Voice"); + + const undone = assignAudioGroup( + { + members: [ + { handle: H1, group: "old" }, + { handle: H2, group: null }, + ], + element: { id: "vo", label: null, present: false }, + }, + resolve, + ); + expect(undone).toEqual({ previous: ["vo", "vo"], changed: true }); + expect(byHandle.get(H1)!.getAttribute("data-audio-group")).toBe("old"); + expect(byHandle.get(H2)!.hasAttribute("data-audio-group")).toBe(false); + expect(document.getElementById("vo")).toBeNull(); + }); + + it("leaves an element that already owns the id", () => { + document.body.innerHTML = `
`; + const resolve = () => null; + for (const present of [true, false]) { + expect( + assignAudioGroup({ members: [], element: { id: "vo", label: null, present } }, resolve), + ).toEqual({ previous: [], changed: false }); + } + expect(document.getElementById("vo")!.localName).toBe("div"); + }); +}); diff --git a/packages/core/src/runtime/editing/audioMeter.ts b/packages/core/src/runtime/editing/audioMeter.ts new file mode 100644 index 0000000000..5864c3a0a3 --- /dev/null +++ b/packages/core/src/runtime/editing/audioMeter.ts @@ -0,0 +1,28 @@ +// `audio-meter`: the preview's program and group level taps, for Studio's meters. `on: true` starts the +// taps when they are off and reads them; `on: false` removes them. +import type { StereoLevel } from "../levelTap"; + +export type AudioMeterParams = { on: boolean }; +export type AudioMeterLevels = { master: StereoLevel; groups: Record }; + +export function parseAudioMeter(params: unknown): AudioMeterParams | null { + if (Object.prototype.toString.call(params) !== "[object Object]") return null; + const { on, ...rest } = params as Record; + return typeof on === "boolean" && Object.keys(rest).length === 0 ? { on } : null; +} + +type LevelTaps = { start(): void; stop(): void; read(): AudioMeterLevels }; + +/** `levels` is null while the taps are off, or when this runtime has no audio graph. */ +export function audioMeter( + { on }: AudioMeterParams, + meter: LevelTaps | undefined, +): { levels: AudioMeterLevels | null } { + if (!meter) return { levels: null }; + if (!on) { + meter.stop(); + return { levels: null }; + } + meter.start(); + return { levels: meter.read() }; +} diff --git a/packages/core/src/runtime/editing/documentMarkup.ts b/packages/core/src/runtime/editing/documentMarkup.ts new file mode 100644 index 0000000000..f0be980067 --- /dev/null +++ b/packages/core/src/runtime/editing/documentMarkup.ts @@ -0,0 +1,3 @@ +export function documentMarkup(): { html: string; baseURI: string } { + return { html: document.documentElement.outerHTML, baseURI: document.baseURI }; +} diff --git a/packages/core/src/runtime/editing/setAudioGroupAttribute.ts b/packages/core/src/runtime/editing/setAudioGroupAttribute.ts new file mode 100644 index 0000000000..95a00b7bfa --- /dev/null +++ b/packages/core/src/runtime/editing/setAudioGroupAttribute.ts @@ -0,0 +1,49 @@ +// `set-audio-group-attribute`: a live write to one ``'s own attribute (fader, mute, label, +// FX chain or automation), which the preview's audio graph follows. Writes no file. +import { HF_AUDIO_AUTOMATION_ATTR } from "../../audioAutomation"; +import { HF_AUDIO_FX_ATTR } from "../../audioFx"; +import { HF_AUDIO_GROUP_TAG } from "../../audioGroups"; + +const GROUP_ATTRIBUTES: readonly string[] = [ + "data-volume", + "data-hidden", + "data-label", + HF_AUDIO_FX_ATTR, + HF_AUDIO_AUTOMATION_ATTR, +]; +const MAX_GROUP_ID = 512; +export const MAX_GROUP_VALUE = 200_000; + +export type SetAudioGroupAttributeParams = { id: string; attr: string; value: string | null }; + +export const isGroupId = (value: unknown): value is string => + typeof value === "string" && value.length > 0 && value.length <= MAX_GROUP_ID; + +export function parseSetAudioGroupAttribute(params: unknown): SetAudioGroupAttributeParams | null { + if (Object.prototype.toString.call(params) !== "[object Object]") return null; + const record = params as Record; + if (Object.keys(record).sort().join() !== "attr,id,value") return null; + const { id, attr, value } = record; + const valid = + isGroupId(id) && + typeof attr === "string" && + GROUP_ATTRIBUTES.includes(attr) && + (value === null || (typeof value === "string" && value.length <= MAX_GROUP_VALUE)); + return valid ? { id, attr, value: value as string | null } : null; +} + +/** The group element `id` names; null when that id is not an ``. */ +export function audioGroupElement(id: string): Element | null { + const el = document.getElementById(id); + return el?.localName === HF_AUDIO_GROUP_TAG ? el : null; +} + +export function setAudioGroupAttribute({ id, attr, value }: SetAudioGroupAttributeParams): { + applied: boolean; +} { + const group = audioGroupElement(id); + if (!group) return { applied: false }; + if (value === null) group.removeAttribute(attr); + else group.setAttribute(attr, value); + return { applied: true }; +} diff --git a/packages/core/src/runtime/editingAgent.test.ts b/packages/core/src/runtime/editingAgent.test.ts index db4158ff93..0487164a5e 100644 --- a/packages/core/src/runtime/editingAgent.test.ts +++ b/packages/core/src/runtime/editingAgent.test.ts @@ -191,6 +191,10 @@ describe("installEditingAgent", () => { "color-grading-previews", "color-grading-playback", "watch-geometry", + "audio-meter", + "document-markup", + "set-audio-group-attribute", + "assign-audio-group", ], }, }); diff --git a/packages/core/src/runtime/editingAgent.ts b/packages/core/src/runtime/editingAgent.ts index af4212ec82..f6855237c3 100644 --- a/packages/core/src/runtime/editingAgent.ts +++ b/packages/core/src/runtime/editingAgent.ts @@ -49,6 +49,10 @@ import { captionTransform } from "./editing/captionTransform"; import { colorGradingStatus } from "./editing/color-grading-status"; import { colorGradingPreviews } from "./editing/color-grading-previews"; import { colorGradingPlayback } from "./editing/color-grading-playback"; +import { assignAudioGroup } from "./editing/assignAudioGroup"; +import { audioMeter } from "./editing/audioMeter"; +import { documentMarkup } from "./editing/documentMarkup"; +import { setAudioGroupAttribute } from "./editing/setAudioGroupAttribute"; type Handlers = { [K in EditingRequestKind]: ( @@ -202,6 +206,10 @@ const HANDLERS: Handlers = { "color-grading-previews": colorGradingPreviews, "color-grading-playback": colorGradingPlayback, "watch-geometry": (params, port) => watchGeometry(port, params, EDITING_PROTOCOL_VERSION), + "audio-meter": (params) => audioMeter(params, window.__hf?.audioMeter), + "document-markup": documentMarkup, + "set-audio-group-attribute": setAudioGroupAttribute, + "assign-audio-group": (params) => assignAudioGroup(params, elementForHandle), }; function serve(port: MessagePort): void { diff --git a/packages/core/src/runtime/editingPort.test.ts b/packages/core/src/runtime/editingPort.test.ts index 192753d52c..727bc39210 100644 --- a/packages/core/src/runtime/editingPort.test.ts +++ b/packages/core/src/runtime/editingPort.test.ts @@ -84,6 +84,10 @@ describe("createEditingPortClient", () => { "color-grading-previews", "color-grading-playback", "watch-geometry", + "audio-meter", + "document-markup", + "set-audio-group-attribute", + "assign-audio-group", ], }); } finally { diff --git a/packages/core/src/runtime/editingPort.ts b/packages/core/src/runtime/editingPort.ts index b064f5a2c3..554422b978 100644 --- a/packages/core/src/runtime/editingPort.ts +++ b/packages/core/src/runtime/editingPort.ts @@ -52,6 +52,12 @@ import { parseColorGradingPlaybackParams, type ColorGradingPlaybackRequest, } from "./editing/color-grading-playback"; +import { parseAssignAudioGroup, type AssignAudioGroupParams } from "./editing/assignAudioGroup"; +import { parseAudioMeter, type AudioMeterLevels } from "./editing/audioMeter"; +import { + parseSetAudioGroupAttribute, + type SetAudioGroupAttributeParams, +} from "./editing/setAudioGroupAttribute"; export { buildStableSelector, @@ -146,6 +152,10 @@ export const EDITING_REQUEST_KINDS = [ "color-grading-previews", "color-grading-playback", "watch-geometry", + "audio-meter", + "document-markup", + "set-audio-group-attribute", + "assign-audio-group", ] as const; export type EditingRequestKind = (typeof EDITING_REQUEST_KINDS)[number]; @@ -262,6 +272,16 @@ export interface EditingRequests { "color-grading-previews": ColorGradingPreviewsRequest; "color-grading-playback": ColorGradingPlaybackRequest; "watch-geometry": { params: WatchGeometryParams; result: { watching: number } }; + "audio-meter": { params: { on: boolean }; result: { levels: AudioMeterLevels | null } }; + "document-markup": { params: NoParams; result: { html: string; baseURI: string } }; + "set-audio-group-attribute": { + params: SetAudioGroupAttributeParams; + result: { applied: boolean }; + }; + "assign-audio-group": { + params: AssignAudioGroupParams; + result: { previous: Array; changed: boolean }; + }; } export type EditingParams = EditingRequests[K]["params"]; export type EditingResult = EditingRequests[K]["result"]; @@ -455,6 +475,10 @@ const PARAM_PARSERS: { "color-grading-previews": parseColorGradingPreviewsParams, "color-grading-playback": parseColorGradingPlaybackParams, "watch-geometry": parseWatchGeometry, + "audio-meter": parseAudioMeter, + "document-markup": noParams, + "set-audio-group-attribute": parseSetAudioGroupAttribute, + "assign-audio-group": parseAssignAudioGroup, }; /** Validates one request off the wire. Never throws: anything unusable becomes a typed error. */ diff --git a/packages/studio/src/components/nle/AudioMeterStrip.tsx b/packages/studio/src/components/nle/AudioMeterStrip.tsx index 5c24c56afa..7417614ada 100644 --- a/packages/studio/src/components/nle/AudioMeterStrip.tsx +++ b/packages/studio/src/components/nle/AudioMeterStrip.tsx @@ -19,6 +19,7 @@ import { } from "@hyperframes/core/audio-gain"; import type { StereoLevel } from "@hyperframes/core/runtime/levelTap"; import { usePlayerStore } from "../../player"; +import { previewFrameOpaque, previewRequest } from "../../player/lib/previewChannel"; import { clampNumber } from "../../utils/studioHelpers"; import { useAudioMetersVisible } from "../../utils/audioMeterVisibility"; import { EXPORT_PEAK_CEILING } from "../../utils/exportPeakCeiling"; @@ -135,6 +136,55 @@ function readHook(iframe: HTMLIFrameElement | null | undefined): AudioMeterHook } } +const isFields = (value: unknown): value is Record => + typeof value === "object" && value !== null; +const stereo = (value: unknown): StereoLevel | undefined => + isFields(value) && Number.isFinite(value.l) && Number.isFinite(value.r) + ? { l: value.l as number, r: value.r as number } + : undefined; + +function meterLevels(value: unknown): Levels | undefined { + if (!isFields(value) || !isFields(value.groups)) return undefined; + const master = stereo(value.master); + if (!master) return undefined; + const groups = Object.entries(value.groups).flatMap(([id, level]) => { + const read = stereo(level); + return read ? [[id, read] as const] : []; + }); + return { master, groups: Object.fromEntries(groups) }; +} + +/** Read interval for a sealed preview's taps: about 30 a second, near the analyser's own window. */ +const SEALED_METER_READ_MS = 33; + +function createSealedMeter() { + let frame: HTMLIFrameElement | null = null; + let levels: Levels | undefined; + let busy = false; + let askedAt = -Infinity; + const stop = () => { + if (frame) void previewRequest(frame, "audio-meter", { on: false }); + frame = null; + levels = undefined; + }; + const read = (iframe: HTMLIFrameElement, now: number): Levels | undefined => { + if (iframe !== frame) stop(); + frame = iframe; + if (busy || now - askedAt < SEALED_METER_READ_MS) return levels; + busy = true; + askedAt = now; + void previewRequest(iframe, "audio-meter", { on: true }) + .then((reply) => { + if (frame === iframe) levels = meterLevels(reply?.levels); + }) + .finally(() => { + busy = false; + }); + return levels; + }; + return { read, stop }; +} + /** 0 dB is bottom 100% plus 1px down, so the tick stays inside overflow-hidden. * A calc() bottom does not stick as an inline style on Windows happy-dom. */ function paintPeak(el: HTMLElement | null, peak: number): void { @@ -218,10 +268,20 @@ function useMeterLoop( let active: AudioMeterHook | null = null; let last = performance.now(); const state = new Map(); + const sealed = createSealedMeter(); + const readLevels = (now: number): Levels | undefined => { + const iframe = previewIframeRef?.current; + if (iframe && previewFrameOpaque(iframe)) { + active = followMeterHook(active, null); + return sealed.read(iframe, now); + } + sealed.stop(); + active = followMeterHook(active, readHook(iframe)); + return active?.read(); + }; const tick = (now: number) => { raf = requestAnimationFrame(tick); - active = followMeterHook(active, readHook(previewIframeRef?.current)); - const levels = active?.read(); + const levels = readLevels(now); const dt = now - last; last = now; evictGoneMeterState(state, new Set(stripsRef.current.map((s) => s.id))); @@ -235,6 +295,7 @@ function useMeterLoop( raf = requestAnimationFrame(tick); return () => { cancelAnimationFrame(raf); + sealed.stop(); try { active?.stop(); } catch { diff --git a/packages/studio/src/components/nle/audioSealedPreview.test.ts b/packages/studio/src/components/nle/audioSealedPreview.test.ts new file mode 100644 index 0000000000..4c48bfb7bb --- /dev/null +++ b/packages/studio/src/components/nle/audioSealedPreview.test.ts @@ -0,0 +1,254 @@ +// @vitest-environment happy-dom + +import { act, createElement } from "react"; +import { createRoot, type Root } from "react-dom/client"; +import { afterEach, beforeEach, expect, it, vi } from "vitest"; +// The frame's real runtime agent, serving the opaque frame from this document. +import { installMarkedEditingAgent } from "../../../../core/src/runtime/editingAgent.test-helpers"; +import { usePlayerStore, type TimelineElement } from "../../player"; +import { ClipMenuAudioItems } from "../../player/components/clipMenuAudioItems"; +import { usePreviewIframeStore } from "../../player/store/previewIframeStore"; +import { TimelineEditProvider } from "../../contexts/TimelineEditContext"; +import { createAudioGroupAndAssignMembers } from "../../hooks/timelineAudioGroupCreate"; +import { mintGroupId } from "../editor/useFxCarveGrouping"; +import { detachedFakeIframe } from "../../player/hooks/timelinePlayerTestHarness"; +import { useAudioMetersVisible } from "../../utils/audioMeterVisibility"; +import { useSetAudioGroupAttribute } from "../../hooks/timelineAudioGroupVolume"; +import { previewDocumentCopy } from "../../player/lib/previewChannel"; +import { AudioMeterStrip } from "./AudioMeterStrip"; + +(globalThis as unknown as { IS_REACT_ACT_ENVIRONMENT: boolean }).IS_REACT_ACT_ENVIRONMENT = true; + +type MeterWindow = Window & { __hf?: Record }; + +const closers: Array<() => void> = []; +const roots: Root[] = []; +let frames: FrameRequestCallback[] = []; +const tick = () => { + const run = frames; + frames = []; + act(() => run.forEach((cb) => cb(performance.now()))); +}; + +beforeEach(() => { + frames = []; + vi.stubGlobal("requestAnimationFrame", (cb: FrameRequestCallback) => frames.push(cb)); + vi.stubGlobal("cancelAnimationFrame", () => {}); + useAudioMetersVisible.setState({ visible: true }); +}); +afterEach(() => { + act(() => roots.splice(0).forEach((root) => root.unmount())); + closers.splice(0).forEach((close) => close()); + delete (window as MeterWindow).__hf; + vi.unstubAllGlobals(); + usePlayerStore.getState().reset(); + usePreviewIframeStore.setState({ iframe: null }); + document.body.innerHTML = ""; +}); + +/** A frame Studio cannot read, whose runtime is the real editing agent over this document. */ +function opaqueFrame() { + const frame: Record = {}; + const iframe = detachedFakeIframe(frame); + Object.defineProperty(iframe, "contentDocument", { value: null }); + Object.defineProperty(frame, "document", { + get: () => { + throw new DOMException("blocked", "SecurityError"); + }, + }); + const parent = { + postMessage: (data: unknown, _origin: string, ports: MessagePort[]) => + window.dispatchEvent( + new MessageEvent("message", { source: frame as unknown as Window, data, ports }), + ), + } as unknown as Window; + const agent = installMarkedEditingAgent(parent); + frame.postMessage = (data: unknown) => + window.dispatchEvent(new MessageEvent("message", { source: parent, data })); + closers.push(() => agent.dispose()); + return iframe; +} + +const member = (domId: string): TimelineElement => ({ + id: domId, + key: `compositions/voices.html#${domId}`, + domId, + tag: "audio", + start: 0, + duration: 5, + track: 0, + audioGroup: "vo", + audioGroupVolume: 1, +}); + +const VOICES = `
+ + +
`; + +function stubProjectFiles(files: Map) { + vi.stubGlobal( + "fetch", + vi.fn(async (input: RequestInfo | URL) => { + const url = String(input); + const content = files.get(decodeURIComponent(url.slice(url.lastIndexOf("/") + 1))); + return new Response(JSON.stringify({ content }), { + status: content === undefined ? 404 : 200, + headers: { "Content-Type": "application/json" }, + }); + }), + ); +} + +it("meter moves + group volume sealed", async () => { + document.body.innerHTML = `
+
${VOICES}
+
`; + const groupEl = document.getElementById("vo")!; + const meter = { + start: vi.fn(), + stop: vi.fn(), + read: vi.fn(() => ({ master: { l: 1, r: 1 }, groups: { vo: { l: 1, r: 1 } } })), + }; + (window as MeterWindow).__hf = { audioMeter: meter }; + usePlayerStore.getState().setElements([member("voice-1")]); + const iframe = opaqueFrame(); + + const host = document.createElement("div"); + document.body.append(host); + const root = createRoot(host); + roots.push(root); + act(() => root.render(createElement(AudioMeterStrip, { previewIframeRef: { current: iframe } }))); + const masks = [...host.querySelectorAll("[data-testid=meter-mask]")]; + expect(masks).toHaveLength(4); + await vi.waitFor(() => { + tick(); + expect(masks.map((mask) => mask.style.height)).toEqual(["0%", "0%", "0%", "0%"]); + }); + expect(meter.start).toHaveBeenCalled(); + act(() => roots.splice(0).forEach((r) => r.unmount())); + await vi.waitFor(() => expect(meter.stop).toHaveBeenCalled()); + + stubProjectFiles(new Map([["compositions/voices.html", VOICES]])); + const writes = new Map(); + const held: { setter: ReturnType | null } = { setter: null }; + function Probe() { + held.setter = useSetAudioGroupAttribute({ + projectIdRef: { current: "project-1" }, + activeCompPath: "index.html", + showToast: vi.fn(), + writeProjectFile: async (path, content) => void writes.set(path, content), + recordEdit: async () => {}, + pendingTimelineEditPathRef: { current: new Set() }, + previewIframeRef: { current: iframe }, + }); + return null; + } + const probeRoot = createRoot(document.createElement("div")); + roots.push(probeRoot); + act(() => probeRoot.render(createElement(Probe))); + const setter = held.setter!; + + setter.setLive("vo", "data-volume", "0.5"); + setter.setLive("vo", "data-volume", "0.4"); + await vi.waitFor(() => expect(groupEl.getAttribute("data-volume")).toBe("0.4")); + expect(usePlayerStore.getState().elements[0]?.audioGroupVolume).toBeCloseTo(0.4, 6); + + await expect(setter.setQuiet("vo", "data-volume", "0.25", "Set volume")).resolves.toEqual({ + status: "saved", + }); + expect([...writes.keys()]).toEqual(["compositions/voices.html"]); + expect(writes.get("compositions/voices.html")).toContain( + '', + ); + await vi.waitFor(() => expect(groupEl.getAttribute("data-volume")).toBe("0.25")); +}); + +it("groups clips in a sealed preview, and unwinds the preview when the save fails", async () => { + document.body.innerHTML = `
+ +
`; + const iframe = opaqueFrame(); + const files = new Map([["index.html", document.body.innerHTML]]); + stubProjectFiles(files); + const elements = ["voice-1", "voice-2"].map((domId) => ({ + ...member(domId), + key: `index.html#${domId}`, + audioGroup: undefined, + })); + const input = { + projectId: "project-1", + activeCompPath: "index.html", + elements, + groupId: "vo", + groupLabel: "Voice", + previewIframe: iframe, + recordEdit: async () => {}, + pendingTimelineEditPathRef: { current: new Set() }, + }; + const groupOf = (id: string) => document.getElementById(id)!.getAttribute("data-audio-group"); + + await expect( + createAudioGroupAndAssignMembers({ + ...input, + writeProjectFile: async () => { + expect([groupOf("voice-1"), groupOf("voice-2")]).toEqual(["vo", "vo"]); + expect(document.querySelector("hf-audio-group#vo")?.getAttribute("data-label")).toBe( + "Voice", + ); + throw new Error("disk full"); + }, + }), + ).rejects.toThrow("disk full"); + expect([groupOf("voice-1"), groupOf("voice-2")]).toEqual(["old", null]); + expect(document.getElementById("vo")).toBeNull(); + + await createAudioGroupAndAssignMembers({ ...input, writeProjectFile: async () => {} }); + expect([groupOf("voice-1"), groupOf("voice-2")]).toEqual(["vo", "vo"]); + expect(document.querySelector("hf-audio-group#vo")).not.toBeNull(); +}); + +it("reads a sealed preview's groups and ids off an inert copy", async () => { + document.body.innerHTML = ``; + const copy = await previewDocumentCopy(opaqueFrame()); + expect(copy).not.toBeNull(); + expect(copy).not.toBe(document); + expect(mintGroupId(copy!)).toBe("voiceover-2"); + expect(copy!.baseURI).toBe(document.baseURI); +}); + +it("offers Duck under voice for a bed in a sealed preview", async () => { + document.body.innerHTML = `
+ + +
`; + const iframe = opaqueFrame(); + usePreviewIframeStore.setState({ iframe }); + usePlayerStore.getState().beginTimelineSession("project-1"); + const bed: TimelineElement = { + id: "music", + domId: "music", + tag: "audio", + start: 0, + duration: 10, + track: 0, + }; + const host = document.createElement("div"); + document.body.append(host); + const root = createRoot(host); + roots.push(root); + const value = { + onSetElementAttributeQuiet: vi.fn(async () => ({ status: "saved" as const })), + onNotice: vi.fn(), + }; + act(() => + root.render( + createElement( + TimelineEditProvider, + { value }, + createElement(ClipMenuAudioItems, { part: "duck", element: bed, onClose: () => {} }), + ), + ), + ); + await vi.waitFor(() => expect(host.textContent).toContain("Duck under voice")); +}); diff --git a/packages/studio/src/hooks/timelineAudioGroupCreate.ts b/packages/studio/src/hooks/timelineAudioGroupCreate.ts index b210c99cd4..5d602fc7d7 100644 --- a/packages/studio/src/hooks/timelineAudioGroupCreate.ts +++ b/packages/studio/src/hooks/timelineAudioGroupCreate.ts @@ -12,6 +12,8 @@ import { saveProjectFilesWithHistory } from "../utils/studioFileHistory"; import { HF_AUDIO_GROUP_ATTR, HF_AUDIO_GROUP_TAG } from "@hyperframes/core/audio-groups"; import { runtimeAudioId } from "../player/lib/timelineElementHelpers"; import { invalidateGroupInfoCache } from "../player/lib/timelineGroupInfo"; +import { previewFrameOpaque, previewRequest } from "../player/lib/previewChannel"; +import { findOpaqueTimelineElement } from "../components/editor/previewSelection"; import { readTagSnippetByTarget, type PatchOperation } from "../utils/sourcePatcher"; import { findTimelineElementInIframe, @@ -146,6 +148,55 @@ function patchLiveGroupElement( return true; } +/** The members' and the group element's live writes, as `patchLiveAudioGroupState` and + * `patchLiveGroupElement` make them; returns their unwind. */ +function patchLiveGroup( + iframe: HTMLIFrameElement | null, + elements: readonly TimelineElement[], + groupId: string, + groupLabel: string | undefined, + activeCompPath: string | null, +): () => void { + const priorGroups = captureAudioGroupState(iframe, elements, activeCompPath); + patchLiveAudioGroupState(iframe, elements, groupId, activeCompPath); + const createdLiveGroupElement = patchLiveGroupElement(iframe, groupId, groupLabel); + return () => { + patchLiveAudioGroupState(iframe, elements, null, activeCompPath, priorGroups); + if (createdLiveGroupElement) { + iframe?.contentDocument?.getElementById(groupId)?.remove(); + } + }; +} + +/** The same writes in a sealed preview, run by its runtime on the members it resolves. */ +async function patchSealedGroup( + iframe: HTMLIFrameElement, + elements: readonly TimelineElement[], + groupId: string, + groupLabel: string | undefined, + activeCompPath: string | null, +): Promise<() => Promise> { + const lookup = { activeCompositionPath: activeCompPath, isMasterView: true }; + const found = await Promise.all( + elements.map((element) => findOpaqueTimelineElement(iframe, element, lookup)), + ); + const handles = found.flatMap((element) => (element ? [element.handle] : [])); + const assigned = await previewRequest(iframe, "assign-audio-group", { + members: handles.map((handle) => ({ handle, group: groupId })), + element: { id: groupId, label: groupLabel ?? null, present: true }, + }); + return async () => { + if (!assigned) return; + await previewRequest(iframe, "assign-audio-group", { + members: handles.map((handle, i) => { + const prior = assigned.previous[i]; + return { handle, group: typeof prior === "string" && prior ? prior : null }; + }), + element: assigned.changed ? { id: groupId, label: null, present: false } : null, + }); + }; +} + interface CreateAudioGroupAndAssignMembersInput { projectId: string; activeCompPath: string | null; @@ -188,9 +239,10 @@ export async function createAudioGroupAndAssignMembers({ throw new Error(`Invalid audio group id ${JSON.stringify(groupId)}`); } - const priorGroups = captureAudioGroupState(previewIframe, elements, activeCompPath); - patchLiveAudioGroupState(previewIframe, elements, groupId, activeCompPath); - const createdLiveGroupElement = patchLiveGroupElement(previewIframe, groupId, groupLabel); + const unwindLive = + previewIframe && previewFrameOpaque(previewIframe) + ? await patchSealedGroup(previewIframe, elements, groupId, groupLabel, activeCompPath) + : patchLiveGroup(previewIframe, elements, groupId, groupLabel, activeCompPath); reseekPreviewRuntime(previewIframe); const groupOperation: PatchOperation = { @@ -220,6 +272,7 @@ export async function createAudioGroupAndAssignMembers({ label: groupLabel ? `Group ${elements.length} clips as ${groupLabel}` : `Group ${elements.length} voice clips`, + // fallow-ignore-next-line code-duplication files, readFile: (path) => readFileContent(projectId, path), writeFile: writeProjectFile, @@ -233,10 +286,7 @@ export async function createAudioGroupAndAssignMembers({ // Mirrors setElementsHidden's failure path: the optimistic live patch // already ran, so a save failure has to be unwound or the preview shows a // grouping that never made it to disk. - patchLiveAudioGroupState(previewIframe, elements, null, activeCompPath, priorGroups); - if (createdLiveGroupElement) { - previewIframe?.contentDocument?.getElementById(groupId)?.remove(); - } + await unwindLive(); reseekPreviewRuntime(previewIframe); throw error; } diff --git a/packages/studio/src/hooks/timelineAudioGroupVolume.ts b/packages/studio/src/hooks/timelineAudioGroupVolume.ts index 0aa120d4ff..72de070eae 100644 --- a/packages/studio/src/hooks/timelineAudioGroupVolume.ts +++ b/packages/studio/src/hooks/timelineAudioGroupVolume.ts @@ -1,4 +1,4 @@ -import { useCallback } from "react"; +import { useCallback, useState } from "react"; import { failedTimelineSave, projectForTimelineSave, @@ -9,6 +9,11 @@ import { HF_AUDIO_AUTOMATION_ATTR } from "@hyperframes/core/audio-automation"; import { usePlayerStore } from "../player"; import type { TimelineElementPatch } from "../player/store/timelineElement"; import { invalidateGroupInfoCache } from "../player/lib/timelineGroupInfo"; +import { + previewDocumentCopy, + previewFrameOpaque, + previewRequest, +} from "../player/lib/previewChannel"; import { buildPatchTarget, persistElementAttribute, @@ -21,6 +26,35 @@ import type { UseTimelineElementVisibilityEditingInput, } from "./timelineTrackVisibility"; +type SealedLane = { busy: boolean; next: { value: string | null } | null }; +const sealedLanes = new WeakMap>(); + +/** A drag writes per frame: one write in flight per group attribute, and only the newest waits. */ +function writeSealedGroupAttribute( + iframe: HTMLIFrameElement, + groupId: string, + attr: string, + value: string | null, +): void { + const lanes = sealedLanes.get(iframe) ?? new Map(); + sealedLanes.set(iframe, lanes); + const key = audioGroupAttributeLiveKey(groupId, attr); + const lane = lanes.get(key) ?? { busy: false, next: null }; + lanes.set(key, lane); + lane.next = { value }; + const pump = () => { + if (lane.busy || !lane.next) return; + const params = { id: groupId, attr, value: lane.next.value }; + lane.next = null; + lane.busy = true; + void previewRequest(iframe, "set-audio-group-attribute", params).finally(() => { + lane.busy = false; + pump(); + }); + }; + pump(); +} + /** Direct DOM write on the group element for the gesture in progress — no * file write, no history entry (mirrors FxParamRow's live/commit split). */ function patchLiveGroupAttribute( @@ -29,6 +63,10 @@ function patchLiveGroupAttribute( attr: string, value: string | null, ): void { + if (iframe && previewFrameOpaque(iframe)) { + writeSealedGroupAttribute(iframe, groupId, attr, value); + return; + } const target = iframe?.contentDocument?.getElementById(groupId); if (!target) return; if (value === null) target.removeAttribute(attr); @@ -146,12 +184,9 @@ export function resolveGroupSourceFile(groupEl: Element | null): string | undefi interface SetAudioGroupAttributeInput { projectId: string; - activeCompPath: string | null; - groupId: string; - attr: string; + source: LiveLaneSource; value: string | null; label: string; - previewIframe: HTMLIFrameElement | null; patchLive: (value: string | null) => void; onFileRead: (value: string | null) => void; writeProjectFile: (path: string, content: string) => Promise; @@ -165,36 +200,31 @@ interface SetAudioGroupAttributeInput { * `createAudioGroupAndAssignMembers`'s save shape but for a single element * and attribute rather than a member-assignment sweep. */ -function groupSaveTarget( - previewIframe: HTMLIFrameElement | null, - groupId: string, - activeCompPath: string | null, -) { - const groupEl = previewIframe?.contentDocument?.getElementById(groupId) ?? null; +function groupSaveTarget(groupEl: Element | null, groupId: string, activeCompPath: string | null) { return { targetPath: resolveGroupSourceFile(groupEl) || activeCompPath || "index.html", patchTarget: buildPatchTarget({ domId: groupId }), }; } +const liveGroupElement = (previewIframe: HTMLIFrameElement | null, groupId: string) => + previewIframe?.contentDocument?.getElementById(groupId) ?? null; + function groupLiveSource( - previewIframe: HTMLIFrameElement | null, + groupEl: Element | null, groupId: string, activeCompPath: string | null, attr: string, ): LiveLaneSource { - const { targetPath, patchTarget } = groupSaveTarget(previewIframe, groupId, activeCompPath); + const { targetPath, patchTarget } = groupSaveTarget(groupEl, groupId, activeCompPath); return { path: targetPath, target: patchTarget, attr }; } async function setAudioGroupAttribute({ projectId, - activeCompPath, - groupId, - attr, + source, value, label, - previewIframe, patchLive, onFileRead, writeProjectFile, @@ -209,7 +239,7 @@ async function setAudioGroupAttribute({ // means `readTagSnippetByTarget` finds nothing and every mute, fader move and // FX preset throws "Unable to patch element in index.html". Every sibling // timeline writer already routes `element.sourceFile || activeCompPath`. - const { targetPath, patchTarget } = groupSaveTarget(previewIframe, groupId, activeCompPath); + const { path: targetPath, target: patchTarget, attr } = source; if (!patchTarget) return null; return persistElementAttribute({ @@ -255,22 +285,62 @@ export function useSetAudioGroupAttribute({ restoreLive: (restore: LiveLaneRestore) => void; } { const liveLanes = useLiveLanes(projectIdRef, activeCompPath); + // A sealed preview's gesture reads the group once, off a copy of the frame, before its first write. + const [sealedReads] = useState(() => new Map>()); + const openSealedLane = useCallback( + (iframe: HTMLIFrameElement, groupId: string, attr: string) => { + const key = audioGroupAttributeLiveKey(groupId, attr); + const open = sealedReads.get(key); + if (open) return open; + const read = previewDocumentCopy(iframe); + sealedReads.set(key, read); + void read.then((doc) => { + if (sealedReads.get(key) !== read) return; + const groupEl = doc?.getElementById(groupId) ?? null; + liveLanes.preview( + key, + () => groupEl?.getAttribute(attr) ?? null, + groupLiveSource(groupEl, groupId, activeCompPath, attr), + ); + }); + return read; + }, + [sealedReads, liveLanes, activeCompPath], + ); + const sealedGroupElement = useCallback( + async (iframe: HTMLIFrameElement, groupId: string, attr: string) => { + const key = audioGroupAttributeLiveKey(groupId, attr); + const read = sealedReads.get(key) ?? previewDocumentCopy(iframe); + const doc = await read; + if (sealedReads.get(key) === read) sealedReads.delete(key); + return doc?.getElementById(groupId) ?? null; + }, + [sealedReads], + ); const setLive = useCallback( (groupId: string, attr: string, value: string | null) => { const key = audioGroupAttributeLiveKey(groupId, attr); - const target = previewIframeRef.current?.contentDocument?.getElementById(groupId); - liveLanes.preview( - key, - () => target?.getAttribute(attr) ?? null, - groupLiveSource(previewIframeRef.current, groupId, activeCompPath, attr), - ); - patchLiveGroupAttribute(previewIframeRef.current, groupId, attr, value); + const iframe = previewIframeRef.current; + if (iframe && previewFrameOpaque(iframe)) { + const read = openSealedLane(iframe, groupId, attr); + void read.then(() => { + if (sealedReads.get(key) === read) patchLiveGroupAttribute(iframe, groupId, attr, value); + }); + } else { + const target = liveGroupElement(iframe, groupId); + liveLanes.preview( + key, + () => target?.getAttribute(attr) ?? null, + groupLiveSource(target, groupId, activeCompPath, attr), + ); + patchLiveGroupAttribute(iframe, groupId, attr, value); + } // Live too, not just on commit: a fader drag is `setLive` per frame and // `setQuiet` once on release, so without this the strip's own readout // fights the drag. syncStoredGroupAttribute(groupId, attr, value); }, - [liveLanes, previewIframeRef, activeCompPath], + [liveLanes, previewIframeRef, activeCompPath, openSealedLane, sealedReads], ); const laneApply = useCallback( (groupId: string, attr: string) => ({ @@ -286,9 +356,12 @@ export function useSetAudioGroupAttribute({ [liveLanes, laneApply], ); const revertLive = useCallback( - (groupId: string, attr: string) => - liveLanes.revert(audioGroupAttributeLiveKey(groupId, attr), laneApply(groupId, attr)), - [liveLanes, laneApply], + (groupId: string, attr: string) => { + const key = audioGroupAttributeLiveKey(groupId, attr); + sealedReads.delete(key); + liveLanes.revert(key, laneApply(groupId, attr)); + }, + [liveLanes, laneApply, sealedReads], ); const setQuiet = useCallback( async ( @@ -301,7 +374,12 @@ export function useSetAudioGroupAttribute({ const pid = projectForTimelineSave(isRecordingRef?.current, project, showToast); // Settles on what the file holds, so overlapping saves that fail cannot leave // the preview or the store on a value that never landed. - const source = groupLiveSource(previewIframeRef.current, groupId, activeCompPath, attr); + const iframe = previewIframeRef.current; + const groupEl = + iframe && previewFrameOpaque(iframe) + ? await sealedGroupElement(iframe, groupId, attr) + : liveGroupElement(iframe, groupId); + const source = groupLiveSource(groupEl, groupId, activeCompPath, attr); const live = claimLive(groupId, attr, source); const unsaved = async (outcome: TimelineEditOutcome): Promise => { live.settle( @@ -313,12 +391,9 @@ export function useSetAudioGroupAttribute({ try { const written = await setAudioGroupAttribute({ projectId: pid, - activeCompPath, - groupId, - attr, + source, value, label, - previewIframe: previewIframeRef.current, patchLive: live.preview, onFileRead: live.read, writeProjectFile, @@ -337,6 +412,7 @@ export function useSetAudioGroupAttribute({ }, [ claimLive, + sealedGroupElement, activeCompPath, previewIframeRef, writeProjectFile, diff --git a/packages/studio/src/player/components/TimelineTrackHeader.tsx b/packages/studio/src/player/components/TimelineTrackHeader.tsx index 09b87a4f91..597d8badde 100644 --- a/packages/studio/src/player/components/TimelineTrackHeader.tsx +++ b/packages/studio/src/player/components/TimelineTrackHeader.tsx @@ -12,6 +12,7 @@ import { useTimelineEditContextOptional } from "../../contexts/TimelineEditConte import { useDomEditActionsContextOptional } from "../../contexts/DomEditContext"; import { mintGroupId } from "../../components/editor/useFxCarveGrouping"; import { runtimeAudioId } from "../lib/timelineElementHelpers"; +import { previewDocumentCopy, previewFrameOpaque } from "../lib/previewChannel"; import { isAudioOnlyTrack } from "../../utils/timelineInspector"; import { TimelineFxButton } from "./TimelineFxButton"; import { getTimelinePropertyLanes } from "./TimelinePropertyLanes"; @@ -262,10 +263,17 @@ export function TimelineTrackHeader({ : null; const canGroupWholeTrack = (allGroupableClipIds?.length ?? 0) >= 2; const groupUngroupedClips = (label: string) => { - const doc = domEditActions?.previewIframeRef.current?.contentDocument; - if (!doc || !onGroupClips) return; - if (!allGroupableClipIds || allGroupableClipIds.length < 2) return; - void onGroupClips(allGroupableClipIds, mintGroupId(doc), label); + const iframe = domEditActions?.previewIframeRef.current; + const ids = allGroupableClipIds; + if (!onGroupClips || !ids || ids.length < 2) return; + if (iframe && previewFrameOpaque(iframe)) { + void previewDocumentCopy(iframe).then( + (doc) => doc && onGroupClips(ids, mintGroupId(doc), label), + ); + return; + } + const doc = iframe?.contentDocument; + if (doc) void onGroupClips(ids, mintGroupId(doc), label); }; return ( diff --git a/packages/studio/src/player/components/clipMenuAudioItems.tsx b/packages/studio/src/player/components/clipMenuAudioItems.tsx index a543593416..f52b273096 100644 --- a/packages/studio/src/player/components/clipMenuAudioItems.tsx +++ b/packages/studio/src/player/components/clipMenuAudioItems.tsx @@ -1,11 +1,21 @@ +import { useEffect, useState } from "react"; +import { HF_AUDIO_GROUP_ATTR } from "@hyperframes/core/audio-groups"; import type { TimelineElement } from "../store/timelineElement"; +import type { TimelineEditCallbacks } from "./timelineCallbacks"; import { useTimelineEditContextOptional } from "../../contexts/TimelineEditContext"; import { useStudioShellContextOptional } from "../../contexts/StudioContext"; import { usePlayerStore } from "../store/playerStore"; import { useLivePreviewIframe } from "../store/previewIframeStore"; import { clipHasSound, throwUnlessSaved, TimelineSaveError } from "./clipMenuNormalize"; import { openAudioGainDialog } from "./audioGainDialogStore"; -import { offersDuck, readBedCarve, setDuckUnderVoice, type DuckOutcome } from "./clipMenuDuck"; +import { + offersDuck, + readBedCarve, + setDuckUnderVoice, + type DuckOutcome, + type GroupClips, +} from "./clipMenuDuck"; +import { previewDocumentCopy, previewFrameOpaque } from "../lib/previewChannel"; const ITEM_CLASS = "w-full flex items-center justify-between px-3 py-1.5 text-xs text-left outline-hidden text-neutral-300 hover:bg-neutral-800 focus-visible:bg-neutral-800 cursor-pointer"; @@ -17,6 +27,38 @@ const DUCK_TOAST: Record = { aborted: "Could not group the voices to duck under.", }; +type Copy = { iframe: HTMLIFrameElement; doc: Document | null }; + +function usePreviewDocument(iframe: HTMLIFrameElement | null): { + doc: Document | null; + copy: Document | null; +} { + const sealed = iframe !== null && previewFrameOpaque(iframe); + const [read, setRead] = useState(null); + useEffect(() => { + if (!iframe || !sealed) return; + let current = true; + void previewDocumentCopy(iframe).then((doc) => current && setRead({ iframe, doc })); + return () => { + current = false; + }; + }, [iframe, sealed]); + if (!sealed) return { doc: iframe?.contentDocument ?? null, copy: null }; + const copy = read?.iframe === iframe ? read.doc : null; + return { doc: copy, copy }; +} + +function voiceGrouper( + onGroupClips: NonNullable, + copy: Document | null, +): GroupClips { + if (!copy) return (ids, groupId) => onGroupClips(ids, groupId, "Voice"); + return async (ids, groupId) => { + await onGroupClips(ids, groupId, "Voice"); + for (const id of ids) copy.getElementById(id)?.setAttribute(HF_AUDIO_GROUP_ATTR, groupId); + }; +} + /** Audio Gain… (opens the G dialog) or Duck under voice (toggle), placed separately in the sound group. */ export function ClipMenuAudioItems({ part, @@ -33,7 +75,8 @@ export function ClipMenuAudioItems({ const sessionProjectId = usePlayerStore((s) => s.timelineProjectId); const projectId = shell?.projectId ?? sessionProjectId; const showToast = shell?.showToast ?? onNotice; - const doc = (shell?.previewIframeRef.current ?? liveIframe)?.contentDocument ?? null; + const iframe = shell?.previewIframeRef.current ?? liveIframe; + const { doc, copy } = usePreviewDocument(iframe); const bed = doc?.getElementById(element.domId ?? element.id) ?? null; const ducked = bed ? readBedCarve(bed)?.enabled === true : false; if (!clipHasSound(element) || !projectId || !showToast || !onSetElementAttributeQuiet) { @@ -52,7 +95,7 @@ export function ClipMenuAudioItems({ await onSetElementAttributeQuiet(element, attr, value, "Duck under voice"), ); }, - onGroupClips ? (ids, groupId) => onGroupClips(ids, groupId, "Voice") : undefined, + onGroupClips && voiceGrouper(onGroupClips, copy), ).catch((error: unknown): DuckOutcome | TimelineSaveError => error instanceof TimelineSaveError ? error : "aborted", ); diff --git a/packages/studio/src/player/lib/previewChannel.ts b/packages/studio/src/player/lib/previewChannel.ts index 7c0c79bca3..dfb13ba722 100644 --- a/packages/studio/src/player/lib/previewChannel.ts +++ b/packages/studio/src/player/lib/previewChannel.ts @@ -124,3 +124,13 @@ export async function previewComputedStyles( } return read; } + +export async function previewDocumentCopy(iframe: HTMLIFrameElement): Promise { + const reply = await previewRequest(iframe, "document-markup", {}); + if (typeof reply?.html !== "string" || typeof reply.baseURI !== "string") return null; + const doc = new DOMParser().parseFromString(reply.html, "text/html"); + const base = doc.createElement("base"); + base.setAttribute("href", reply.baseURI); + doc.head.prepend(base); + return doc; +} diff --git a/scripts/package-size-budgets.json b/scripts/package-size-budgets.json index 7c40464707..6934a3f9d7 100644 --- a/scripts/package-size-budgets.json +++ b/scripts/package-size-budgets.json @@ -21,7 +21,7 @@ }, "@hyperframes/inspector": { "packed": 933584, - "unpacked": 3830000 + "unpacked": 4010650 }, "@hyperframes/lint": { "packed": 191022,