From d7d416124f46a3c10066adba86be715907425762 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81ngel?= Date: Tue, 17 Nov 2026 07:00:14 +0000 Subject: [PATCH] fix(studio): colour grading works in a sealed preview (#7999) * test(studio): colour grading previews and status on a sealed preview * fix(studio): colour grading reads a sealed preview through the editing port --- .fallowrc.jsonc | 3 + .../runtime/editing/color-grading-playback.ts | 29 +++ .../runtime/editing/color-grading-previews.ts | 67 +++++++ .../runtime/editing/color-grading-status.ts | 72 ++++++++ .../src/runtime/editing/color-grading.test.ts | 130 ++++++++++++++ .../core/src/runtime/editingAgent.test.ts | 3 + packages/core/src/runtime/editingAgent.ts | 6 + packages/core/src/runtime/editingPort.test.ts | 3 + packages/core/src/runtime/editingPort.ts | 21 +++ .../useColorGradingController.sealed.test.ts | 165 +++++++++++++++++ .../editor/useColorGradingController.ts | 21 ++- .../editor/useColorGradingPreviews.ts | 170 +++++++++++++----- scripts/package-size-budgets.json | 2 +- 13 files changed, 647 insertions(+), 45 deletions(-) create mode 100644 packages/core/src/runtime/editing/color-grading-playback.ts create mode 100644 packages/core/src/runtime/editing/color-grading-previews.ts create mode 100644 packages/core/src/runtime/editing/color-grading-status.ts create mode 100644 packages/core/src/runtime/editing/color-grading.test.ts create mode 100644 packages/studio/src/components/editor/useColorGradingController.sealed.test.ts diff --git a/.fallowrc.jsonc b/.fallowrc.jsonc index 9ae69f725b..f13d626a34 100644 --- a/.fallowrc.jsonc +++ b/.fallowrc.jsonc @@ -1065,6 +1065,9 @@ "packages/inspector/src/timelineReport.ts", "packages/inspector/src/pageQueries.ts", "packages/core/src/runtime/timelineValues.ts", + // useColorGradingController.ts: the hook's complexity is inherited; the sealed-preview + // status read adds no branch to it, but its import lines shift the hook's fingerprint. + "packages/studio/src/components/editor/useColorGradingController.ts", ], }, } diff --git a/packages/core/src/runtime/editing/color-grading-playback.ts b/packages/core/src/runtime/editing/color-grading-playback.ts new file mode 100644 index 0000000000..a209f4f6fc --- /dev/null +++ b/packages/core/src/runtime/editing/color-grading-playback.ts @@ -0,0 +1,29 @@ +import type { HfColorGradingTarget } from "../../colorGrading"; +import { colorGradingTarget, gradingRuntime, isRecord } from "./color-grading-status"; + +/** Plays the target video muted and looping for a live preview; null stops the one playing. */ +export type ColorGradingPlaybackRequest = { + params: { target: HfColorGradingTarget | null }; + result: { started: boolean }; +}; + +// One preview plays at a time, as Studio hovers one card at a time. +let stopPlaying: (() => void) | null = null; + +export function parseColorGradingPlaybackParams( + params: unknown, +): ColorGradingPlaybackRequest["params"] | null { + if (!isRecord(params) || Object.keys(params).join() !== "target") return null; + const { target } = params; + if (target === null) return { target }; + const parsed = colorGradingTarget(target); + return parsed ? { target: parsed } : null; +} + +export function colorGradingPlayback({ + target, +}: ColorGradingPlaybackRequest["params"]): ColorGradingPlaybackRequest["result"] { + stopPlaying?.(); + stopPlaying = target ? (gradingRuntime()?.startPreviewPlayback(target) ?? null) : null; + return { started: stopPlaying !== null }; +} diff --git a/packages/core/src/runtime/editing/color-grading-previews.ts b/packages/core/src/runtime/editing/color-grading-previews.ts new file mode 100644 index 0000000000..2fddb27e79 --- /dev/null +++ b/packages/core/src/runtime/editing/color-grading-previews.ts @@ -0,0 +1,67 @@ +import type { HfColorGradingTarget } from "../../colorGrading"; +import { + colorGradingTarget, + gradingRuntime, + isRecord, + type GradingPreviewBatch, + type GradingPreviewCandidate, +} from "./color-grading-status"; + +const MAX_GRADING_CANDIDATES = 64; +const MAX_CANDIDATE_ID = 64; +/** The runtime draws previews no larger than this on their long side. */ +const MAX_PREVIEW_DIMENSION = 320; + +/** Each candidate drawn over the element's current frame; null when nothing could be drawn. */ +export type ColorGradingPreviewsRequest = { + params: { + target: HfColorGradingTarget; + candidates: GradingPreviewCandidate[]; + maxDimension: number; + useMediaTime: boolean; + }; + result: { batch: GradingPreviewBatch | null }; +}; + +function candidate(value: unknown): GradingPreviewCandidate | null { + if (!isRecord(value) || Object.keys(value).sort().join() !== "grading,id") return null; + const { id, grading } = value; + if (typeof id !== "string" || !id || id.length > MAX_CANDIDATE_ID) return null; + return grading === null || isRecord(grading) ? { id, grading } : null; +} + +export function parseColorGradingPreviewsParams( + params: unknown, +): ColorGradingPreviewsRequest["params"] | null { + if (!isRecord(params)) return null; + if (Object.keys(params).sort().join() !== "candidates,maxDimension,target,useMediaTime") { + return null; + } + const { candidates, maxDimension, useMediaTime } = params; + const target = colorGradingTarget(params.target); + if (!target || typeof useMediaTime !== "boolean" || !Array.isArray(candidates)) return null; + if (!candidates.length || candidates.length > MAX_GRADING_CANDIDATES) return null; + const valid = Number.isFinite(maxDimension) && (maxDimension as number) > 0; + if (!valid || (maxDimension as number) > MAX_PREVIEW_DIMENSION) return null; + const parsed = candidates.map(candidate); + if (parsed.includes(null)) return null; + return { + target, + candidates: parsed as GradingPreviewCandidate[], + maxDimension: maxDimension as number, + useMediaTime, + }; +} + +export async function colorGradingPreviews({ + target, + candidates, + maxDimension, + useMediaTime, +}: ColorGradingPreviewsRequest["params"]): Promise { + const runtime = gradingRuntime(); + if (!runtime) return { batch: null }; + return { + batch: await runtime.renderPreviews(target, candidates, { maxDimension, useMediaTime }), + }; +} diff --git a/packages/core/src/runtime/editing/color-grading-status.ts b/packages/core/src/runtime/editing/color-grading-status.ts new file mode 100644 index 0000000000..2f198e7689 --- /dev/null +++ b/packages/core/src/runtime/editing/color-grading-status.ts @@ -0,0 +1,72 @@ +import type { HfColorGradingTarget } from "../../colorGrading"; + +/** The page's colour grading runtime as these kinds use it, declared here so the port does not pull + * runtime/colorGrading.ts into every build that imports it. */ +interface GradingRuntime { + getStatus(target: HfColorGradingTarget): GradingStatus; + renderPreviews( + target: HfColorGradingTarget, + candidates: readonly GradingPreviewCandidate[], + options: { maxDimension: number; useMediaTime: boolean }, + ): Promise; + startPreviewPlayback(target: HfColorGradingTarget): (() => void) | null; +} +interface GradingStatus { + state: "missing" | "inactive" | "pending" | "active" | "unavailable"; + message: string; +} +export interface GradingPreviewCandidate { + id: string; + grading: unknown; +} +export interface GradingPreviewBatch { + width: number; + height: number; + images: Array<{ id: string; dataUrl: string | null; error?: string }>; +} + +const MAX_TARGET_TEXT = 2_000; +const TARGET_KEYS = "hfId,id,selector,selectorIndex"; + +// editingPort.ts imports this file, so taking its isPlainRecord would make a cycle. +export const isRecord = (value: unknown): value is Record => + Object.prototype.toString.call(value) === "[object Object]"; + +/** The page's colour grading runtime, when it has one. */ +export const gradingRuntime = (): GradingRuntime | undefined => + (Reflect.get(window, "__hf") as { colorGrading?: GradingRuntime } | undefined)?.colorGrading; + +const isTargetText = (value: unknown) => + value === null || (typeof value === "string" && value.length <= MAX_TARGET_TEXT); + +/** The media element a colour grading request names, in the shape Studio sends: every key, each nullable. */ +export function colorGradingTarget(value: unknown): HfColorGradingTarget | null { + if (!isRecord(value) || Object.keys(value).sort().join() !== TARGET_KEYS) return null; + const { id, hfId, selector, selectorIndex } = value; + const index = + selectorIndex === null || + (Number.isSafeInteger(selectorIndex) && (selectorIndex as number) >= 0); + return isTargetText(id) && isTargetText(hfId) && isTargetText(selector) && index + ? (value as HfColorGradingTarget) + : null; +} + +/** The grading state of one media element; null when the page has no colour grading runtime. */ +export type ColorGradingStatusRequest = { + params: { target: HfColorGradingTarget }; + result: { status: GradingStatus | null }; +}; + +export function parseColorGradingStatusParams( + params: unknown, +): ColorGradingStatusRequest["params"] | null { + if (!isRecord(params) || Object.keys(params).join() !== "target") return null; + const target = colorGradingTarget(params.target); + return target ? { target } : null; +} + +export function colorGradingStatus({ + target, +}: ColorGradingStatusRequest["params"]): ColorGradingStatusRequest["result"] { + return { status: gradingRuntime()?.getStatus(target) ?? null }; +} diff --git a/packages/core/src/runtime/editing/color-grading.test.ts b/packages/core/src/runtime/editing/color-grading.test.ts new file mode 100644 index 0000000000..4642d427ec --- /dev/null +++ b/packages/core/src/runtime/editing/color-grading.test.ts @@ -0,0 +1,130 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { installEditingAgent } from "../editingAgent"; + +const CONNECT = { source: "hf-parent", type: "editing-connect", v: 1 }; +const target = { id: "clip", hfId: null, selector: "#clip", selectorIndex: 0 }; + +let agent: { dispose: () => void } | null = null; +afterEach(() => { + agent?.dispose(); + agent = null; + delete window.__hf; + document.body.innerHTML = ""; +}); + +function connect(): MessagePort { + const parent = document.body.appendChild(document.createElement("iframe")).contentWindow!; + const handoff = vi.fn(); + parent.postMessage = handoff as unknown as Window["postMessage"]; + agent = installEditingAgent(parent); + window.dispatchEvent(new MessageEvent("message", { source: parent, data: CONNECT })); + return (handoff.mock.calls[0]![2] as MessagePort[])[0]!; +} + +let nextId = 1; +function ask(port: MessagePort, kind: string, params: unknown): Promise> { + return new Promise((resolve) => { + port.onmessage = (event) => resolve(event.data); + port.postMessage({ v: 1, id: nextId++, kind, params }); + }); +} + +function installGrading() { + const stop = vi.fn(); + const grading = { + getStatus: vi.fn(() => ({ state: "active", message: "Shader active" })), + renderPreviews: vi.fn(async () => ({ + width: 160, + height: 90, + images: [{ id: "warm", dataUrl: "data:image/png;base64,x" }], + })), + startPreviewPlayback: vi.fn(() => stop), + }; + window.__hf = { colorGrading: grading as never }; + return { grading, stop }; +} + +describe("colour grading kinds", () => { + it("answers with the frame's grading runtime", async () => { + const { grading, stop } = installGrading(); + const port = connect(); + const candidates = [{ id: "warm", grading: { preset: "warm-daylight" } }]; + + expect((await ask(port, "color-grading-status", { target })).result).toEqual({ + status: { state: "active", message: "Shader active" }, + }); + const previews = await ask(port, "color-grading-previews", { + target, + candidates, + maxDimension: 160, + useMediaTime: true, + }); + expect(previews.result).toEqual({ + batch: { + width: 160, + height: 90, + images: [{ id: "warm", dataUrl: "data:image/png;base64,x" }], + }, + }); + expect(grading.renderPreviews).toHaveBeenCalledWith(target, candidates, { + maxDimension: 160, + useMediaTime: true, + }); + + expect((await ask(port, "color-grading-playback", { target })).result).toEqual({ + started: true, + }); + expect(grading.startPreviewPlayback).toHaveBeenCalledWith(target); + expect((await ask(port, "color-grading-playback", { target: null })).result).toEqual({ + started: false, + }); + expect(stop).toHaveBeenCalledOnce(); + }); + + it("answers null when the page has no grading runtime", async () => { + const port = connect(); + expect((await ask(port, "color-grading-status", { target })).result).toEqual({ status: null }); + const previews = await ask(port, "color-grading-previews", { + target, + candidates: [{ id: "a", grading: null }], + maxDimension: 160, + useMediaTime: false, + }); + expect(previews.result).toEqual({ batch: null }); + expect((await ask(port, "color-grading-playback", { target })).result).toEqual({ + started: false, + }); + }); + + const previews = { target, candidates: [{ id: "a", grading: null }], maxDimension: 160 }; + it.each([ + ["color-grading-status", {}], + ["color-grading-status", { target: "#clip" }], + ["color-grading-status", { target: { id: "clip" } }], + ["color-grading-status", { target: { ...target, selector: "p".repeat(2001) } }], + ["color-grading-status", { target: { ...target, selectorIndex: -1 } }], + ["color-grading-status", { target, extra: 1 }], + ["color-grading-previews", { ...previews }], + ["color-grading-previews", { ...previews, useMediaTime: false, maxDimension: 321 }], + ["color-grading-previews", { ...previews, useMediaTime: false, maxDimension: 0 }], + ["color-grading-previews", { ...previews, useMediaTime: false, candidates: [] }], + [ + "color-grading-previews", + { ...previews, useMediaTime: false, candidates: Array(65).fill({ id: "a", grading: null }) }, + ], + ["color-grading-previews", { ...previews, useMediaTime: false, candidates: [{ id: "a" }] }], + [ + "color-grading-previews", + { ...previews, useMediaTime: false, candidates: [{ id: "a", grading: "warm" }] }, + ], + ["color-grading-playback", {}], + ["color-grading-playback", { target: { id: 1 } }], + ])("rejects %s %j", async (kind, params) => { + const { grading } = installGrading(); + const reply = await ask(connect(), kind, params); + expect(reply).toMatchObject({ ok: false, error: { code: "invalid-params" } }); + expect(grading.getStatus).not.toHaveBeenCalled(); + expect(grading.renderPreviews).not.toHaveBeenCalled(); + expect(grading.startPreviewPlayback).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/core/src/runtime/editingAgent.test.ts b/packages/core/src/runtime/editingAgent.test.ts index 7cf0a6a772..4138f5966d 100644 --- a/packages/core/src/runtime/editingAgent.test.ts +++ b/packages/core/src/runtime/editingAgent.test.ts @@ -172,6 +172,9 @@ describe("installEditingAgent", () => { "restore-attributes", "caption-words", "caption-transform", + "color-grading-status", + "color-grading-previews", + "color-grading-playback", ], }, }); diff --git a/packages/core/src/runtime/editingAgent.ts b/packages/core/src/runtime/editingAgent.ts index 78a84d84cb..4e1685a30e 100644 --- a/packages/core/src/runtime/editingAgent.ts +++ b/packages/core/src/runtime/editingAgent.ts @@ -45,6 +45,9 @@ import { matchCombo, parseCombo, type Combo } from "../keyCombo"; import { shouldIgnorePlaybackShortcutTarget } from "../typingTarget"; import { captionWords } from "./editing/captionWords"; import { captionTransform } from "./editing/captionTransform"; +import { colorGradingStatus } from "./editing/color-grading-status"; +import { colorGradingPreviews } from "./editing/color-grading-previews"; +import { colorGradingPlayback } from "./editing/color-grading-playback"; type Handlers = { [K in EditingRequestKind]: ( @@ -194,6 +197,9 @@ const HANDLERS: Handlers = { "restore-attributes": (params) => restoreAttributes(params), "caption-words": captionWords, "caption-transform": captionTransform, + "color-grading-status": colorGradingStatus, + "color-grading-previews": colorGradingPreviews, + "color-grading-playback": colorGradingPlayback, }; function serve(port: MessagePort): void { diff --git a/packages/core/src/runtime/editingPort.test.ts b/packages/core/src/runtime/editingPort.test.ts index 0ed526461d..9bb1f27349 100644 --- a/packages/core/src/runtime/editingPort.test.ts +++ b/packages/core/src/runtime/editingPort.test.ts @@ -80,6 +80,9 @@ describe("createEditingPortClient", () => { "restore-attributes", "caption-words", "caption-transform", + "color-grading-status", + "color-grading-previews", + "color-grading-playback", ], }); } finally { diff --git a/packages/core/src/runtime/editingPort.ts b/packages/core/src/runtime/editingPort.ts index 62b8aa25f1..05798d126e 100644 --- a/packages/core/src/runtime/editingPort.ts +++ b/packages/core/src/runtime/editingPort.ts @@ -34,6 +34,18 @@ import { type CaptionTransformParams, type CaptionWordState, } from "./editing/captionTransform"; +import { + parseColorGradingStatusParams, + type ColorGradingStatusRequest, +} from "./editing/color-grading-status"; +import { + parseColorGradingPreviewsParams, + type ColorGradingPreviewsRequest, +} from "./editing/color-grading-previews"; +import { + parseColorGradingPlaybackParams, + type ColorGradingPlaybackRequest, +} from "./editing/color-grading-playback"; export { buildStableSelector, @@ -124,6 +136,9 @@ export const EDITING_REQUEST_KINDS = [ "restore-attributes", "caption-words", "caption-transform", + "color-grading-status", + "color-grading-previews", + "color-grading-playback", ] as const; export type EditingRequestKind = (typeof EDITING_REQUEST_KINDS)[number]; @@ -236,6 +251,9 @@ export interface EditingRequests { /** Boxes of the words of visible caption groups; a group written as plain text is split into word spans. */ "caption-words": { params: CaptionWordsParams; result: { boxes: CaptionWordBox[] } }; "caption-transform": { params: CaptionTransformParams; result: { words: CaptionWordState[] } }; + "color-grading-status": ColorGradingStatusRequest; + "color-grading-previews": ColorGradingPreviewsRequest; + "color-grading-playback": ColorGradingPlaybackRequest; } export type EditingParams = EditingRequests[K]["params"]; export type EditingResult = EditingRequests[K]["result"]; @@ -425,6 +443,9 @@ const PARAM_PARSERS: { "restore-attributes": parseRestoreAttributes, "caption-words": captionWordsParams, "caption-transform": captionTransformParams, + "color-grading-status": parseColorGradingStatusParams, + "color-grading-previews": parseColorGradingPreviewsParams, + "color-grading-playback": parseColorGradingPlaybackParams, }; /** Validates one request off the wire. Never throws: anything unusable becomes a typed error. */ diff --git a/packages/studio/src/components/editor/useColorGradingController.sealed.test.ts b/packages/studio/src/components/editor/useColorGradingController.sealed.test.ts new file mode 100644 index 0000000000..6a0e251e36 --- /dev/null +++ b/packages/studio/src/components/editor/useColorGradingController.sealed.test.ts @@ -0,0 +1,165 @@ +// @vitest-environment happy-dom + +import React, { act } from "react"; +import { createRoot } from "react-dom/client"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { normalizeHfColorGrading } from "@hyperframes/core/color-grading"; +// The frame's real runtime agent, serving the opaque frame from this document. +import { installEditingAgent } from "../../../../core/src/runtime/editingAgent"; +import { installReactActEnvironment } from "../../hooks/domSelectionTestHarness"; +import { detachedFakeIframe } from "../../player/hooks/timelinePlayerTestHarness"; +import type { DomEditSelection } from "./domEditing"; +import { useColorGradingController } from "./useColorGradingController"; + +installReactActEnvironment(); + +const closers: Array<() => void> = []; +afterEach(() => { + closers.splice(0).forEach((close) => close()); + delete (window as { __hf?: unknown }).__hf; + document.body.innerHTML = ""; +}); + +/** A frame Studio cannot read, whose runtime is the real editing agent over this document. */ +function opaqueFrame(): HTMLIFrameElement { + const frame: Record = {}; + const iframe = detachedFakeIframe(frame); + Object.defineProperty(iframe, "contentDocument", { value: null }); + Object.defineProperty(frame, "document", { + get: () => { + throw new DOMException("blocked", "SecurityError"); + }, + }); + const parent = { + postMessage: (data: unknown, _origin: string, ports: MessagePort[]) => + window.dispatchEvent( + new MessageEvent("message", { source: frame as unknown as Window, data, ports }), + ), + } as unknown as Window; + const agent = installEditingAgent(parent); + frame.postMessage = (data: unknown) => + window.dispatchEvent(new MessageEvent("message", { source: parent, data })); + closers.push(() => agent.dispose()); + return iframe; +} + +/** The frame's colour grading runtime, on the window the agent serves. */ +function installFrameGrading() { + const stopPlayback = vi.fn(); + const grading = { + getStatus: vi.fn(() => ({ state: "active", message: "Shader active" })), + renderPreviews: vi.fn(async (_target: unknown, candidates: Array<{ id: string }>) => ({ + width: 160, + height: 90, + images: candidates.map(({ id }) => ({ + id, + dataUrl: id === "neutral" ? "https://example.com/x.png" : `data:image/png;base64,${id}`, + })), + })), + startPreviewPlayback: vi.fn(() => stopPlayback), + }; + Object.assign(window, { __hf: { colorGrading: grading } }); + return { grading, stopPlayback }; +} + +const element = { + element: document.createElement("video"), + id: "s1-bg", + hfId: "hf-bg", + selector: "#s1-bg", + selectorIndex: 0, + tagName: "video", + sourceFile: "index.html", + dataAttributes: {}, +} as unknown as DomEditSelection; +const target = { id: "s1-bg", hfId: "hf-bg", selector: "#s1-bg", selectorIndex: 0 }; + +function renderController(iframe: HTMLIFrameElement) { + let latest: ReturnType | undefined; + const previewIframeRef = { current: iframe }; + const onSetAttributeLive = vi.fn(); + function Host() { + latest = useColorGradingController({ + projectId: "proj", + element, + previewIframeRef, + onSetAttributeLive, + }); + return null; + } + const root = createRoot(document.body.appendChild(document.createElement("div"))); + act(() => root.render(React.createElement(Host))); + closers.unshift(() => act(() => root.unmount())); + return () => { + if (!latest) throw new Error("hook did not render"); + return latest; + }; +} + +// Each poll flushes React's queued updates first; a check inside one act would never see them. +const settle = (check: () => void) => + vi.waitFor( + async () => { + await act(async () => {}); + check(); + }, + { timeout: 3_000 }, + ); + +describe("grade preview sealed", () => { + it("reads the frame's grading status over the editing port", async () => { + const { grading } = installFrameGrading(); + const state = renderController(opaqueFrame()); + + await settle(() => + expect(state().runtimeStatus).toEqual({ state: "active", message: "Shader active" }), + ); + expect(grading.getStatus).toHaveBeenCalledWith(target); + }); + + it("draws preset cards the frame renders, and drops any that are not image data", async () => { + const { grading } = installFrameGrading(); + const state = renderController(opaqueFrame()); + + act(() => state().requestPresetPreviews()); + await settle(() => expect(state().presetPreviews.images["bright-pop"]).toBeDefined()); + + const [renderedFor, candidates, options] = grading.renderPreviews.mock.calls[0]!; + expect(renderedFor).toEqual(target); + expect(candidates).toHaveLength(18); + expect(options).toMatchObject({ maxDimension: 160 }); + expect(state().presetPreviews).toMatchObject({ width: 160, height: 90 }); + expect(state().presetPreviews.images["bright-pop"]).toBe("data:image/png;base64,bright-pop"); + expect(state().presetPreviews.images.neutral).toBeUndefined(); + }); + + it("plays the hovered card in the frame and stops it when the hover ends", async () => { + const { grading, stopPlayback } = installFrameGrading(); + const state = renderController(opaqueFrame()); + const look = normalizeHfColorGrading({ preset: "bright-pop", intensity: 1 })!; + + act(() => + state().previewColorGrading(look, { animatedPreview: { kind: "presets", id: "bright-pop" } }), + ); + await settle(() => expect(grading.renderPreviews).toHaveBeenCalled()); + expect(grading.startPreviewPlayback).toHaveBeenCalledWith(target); + expect(grading.renderPreviews.mock.calls[0]![2]).toEqual({ + maxDimension: 160, + useMediaTime: true, + }); + await settle(() => + expect(state().presetPreviews.images["bright-pop"]).toBe("data:image/png;base64,bright-pop"), + ); + + act(() => state().previewColorGrading(null)); + await settle(() => expect(stopPlayback).toHaveBeenCalledTimes(1)); + }); + + it("captures a graded frame from the frame", async () => { + installFrameGrading(); + const state = renderController(opaqueFrame()); + + const frame = await act(() => state().captureGradedFrame()); + expect(frame).toEqual({ dataUrl: "data:image/png;base64,capture", width: 160, height: 90 }); + }); +}); diff --git a/packages/studio/src/components/editor/useColorGradingController.ts b/packages/studio/src/components/editor/useColorGradingController.ts index 084991808a..8d70331ccb 100644 --- a/packages/studio/src/components/editor/useColorGradingController.ts +++ b/packages/studio/src/components/editor/useColorGradingController.ts @@ -24,6 +24,7 @@ import { postRuntimeControlMessage, } from "../../player/lib/runtimeProtocol"; import { + readSealedColorGradingStatus, useColorGradingPreviews, type ColorGradingCapturedFrame, type ColorGradingPresetPreviews, @@ -31,6 +32,7 @@ import { } from "./useColorGradingPreviews"; import { studioApiFetch } from "../../utils/studioApiFetch"; import { authoredSrcPath } from "../../player/components/thumbnailUtils"; +import { previewFrameOpaque } from "../../player/lib/previewChannel"; export type { ColorGradingPresetPreviews, ColorGradingPreviewOptions }; @@ -279,8 +281,9 @@ export function useColorGradingController({ [element.hfId, element.id, element.selector, element.selectorIndex], ); + const statusReadRef = useRef(0); const refreshRuntimeStatus = useCallback(() => { - setRuntimeStatus(readRuntimeColorGradingStatus(previewIframeRef?.current, target)); + refreshStatus(previewIframeRef?.current, target, statusReadRef, setRuntimeStatus); }, [previewIframeRef, target]); useEffect(() => { @@ -568,3 +571,19 @@ export function useColorGradingController({ }, }; } + +function refreshStatus( + iframe: HTMLIFrameElement | null | undefined, + target: HfColorGradingTarget, + reads: { current: number }, + apply: (status: RuntimeColorGradingStatus) => void, +): void { + if (!iframe || !previewFrameOpaque(iframe)) { + apply(readRuntimeColorGradingStatus(iframe, target)); + return; + } + const read = ++reads.current; + void readSealedColorGradingStatus(iframe, target).then((status) => { + if (read === reads.current) apply(status); + }); +} diff --git a/packages/studio/src/components/editor/useColorGradingPreviews.ts b/packages/studio/src/components/editor/useColorGradingPreviews.ts index bc1aa33c59..66068e0790 100644 --- a/packages/studio/src/components/editor/useColorGradingPreviews.ts +++ b/packages/studio/src/components/editor/useColorGradingPreviews.ts @@ -1,4 +1,6 @@ import { useLivePreviewIframe } from "../../player/store/previewIframeStore"; +import { previewFrameOpaque, previewRequest } from "../../player/lib/previewChannel"; +import type { EditingResult } from "@hyperframes/core/runtime/editing-port"; import { useCallback, useEffect, useRef, useState, type RefObject } from "react"; import { HF_COLOR_GRADING_ACTIVE_EFFECT_KEYS, @@ -29,19 +31,23 @@ export interface ColorGradingPreviewOptions { animatedPreview: { kind: ColorGradingPreviewKind; id: string }; } +type PreviewBatch = { + width: number; + height: number; + images: Array<{ id: string; dataUrl: string | null }>; +}; + type RuntimeColorGradingPreview = { renderPreviews: ( target: HfColorGradingTarget, candidates: Array<{ id: string; grading: unknown }>, options: { maxDimension: number; useMediaTime?: boolean }, - ) => Promise<{ - width: number; - height: number; - images: Array<{ id: string; dataUrl: string | null }>; - } | null>; + ) => Promise; startPreviewPlayback?: (target: HfColorGradingTarget) => (() => void) | null; }; +type AnimatedSession = { stopPlayback: () => void; timer: number | null }; + type PreviewRequest = { kind: ColorGradingPreviewKind; version: number; @@ -74,6 +80,79 @@ function readRuntime( } } +const PORT_PREVIEW_TIMEOUT_MS = 15_000; + +const isImageData = (value: unknown): value is string => + typeof value === "string" && value.startsWith("data:image/"); + +function portBatch(batch: unknown): PreviewBatch | null { + if (typeof batch !== "object" || batch === null) return null; + const { width, height, images } = batch as Record; + if (!Number.isFinite(width) || !Number.isFinite(height) || !Array.isArray(images)) return null; + return { + width: width as number, + height: height as number, + images: images.map((image: unknown) => { + const { id, dataUrl } = (image ?? {}) as { id?: unknown; dataUrl?: unknown }; + return { id: String(id), dataUrl: isImageData(dataUrl) ? dataUrl : null }; + }), + }; +} + +function portRuntime(iframe: HTMLIFrameElement): RuntimeColorGradingPreview { + return { + renderPreviews: async (target, candidates, { maxDimension, useMediaTime = false }) => { + const params = { target, candidates, maxDimension, useMediaTime }; + const reply = await previewRequest( + iframe, + "color-grading-previews", + params, + PORT_PREVIEW_TIMEOUT_MS, + ); + return portBatch(reply?.batch); + }, + }; +} + +function previewRuntime( + iframe: HTMLIFrameElement | null | undefined, +): RuntimeColorGradingPreview | null { + return iframe && previewFrameOpaque(iframe) ? portRuntime(iframe) : readRuntime(iframe); +} + +function playInSealedFrame( + iframe: HTMLIFrameElement | null | undefined, + target: HfColorGradingTarget, + sessionRef: { current: AnimatedSession | null }, + animate: (runtime: RuntimeColorGradingPreview, session: AnimatedSession) => void, +): boolean { + if (!iframe || !previewFrameOpaque(iframe)) return false; + const stopPlayback = () => + void previewRequest(iframe, "color-grading-playback", { target: null }); + const session: AnimatedSession = { stopPlayback, timer: null }; + sessionRef.current = session; + void previewRequest(iframe, "color-grading-playback", { target }).then((reply) => { + if (sessionRef.current !== session) return; + if (reply?.started) animate(portRuntime(iframe), session); + else sessionRef.current = null; + }); + return true; +} + +type ColorGradingStatus = NonNullable["status"]>; +const STATUS_STATES = new Set(["missing", "inactive", "pending", "active", "unavailable"]); + +export async function readSealedColorGradingStatus( + iframe: HTMLIFrameElement, + target: HfColorGradingTarget, +): Promise { + const reply = await previewRequest(iframe, "color-grading-status", { target }); + const { state, message } = (reply?.status ?? {}) as Partial>; + return STATUS_STATES.has(state as string) && typeof message === "string" + ? { state: state as ColorGradingStatus["state"], message } + : { state: "pending", message: "Waiting for runtime" }; +} + function previewMaxDimension(): number { return Math.min(320, Math.ceil(160 * Math.max(1, window.devicePixelRatio || 1))); } @@ -149,7 +228,7 @@ export function useColorGradingPreviews({ previews: emptyPreviews(), })); const [request, setRequest] = useState(null); - const animatedRef = useRef<{ stopPlayback: () => void; timer: number | null } | null>(null); + const animatedRef = useRef(null); if (state.identityKey !== identityKey) { setState({ identityKey, previews: emptyPreviews() }); setRequest(null); @@ -186,7 +265,7 @@ export function useColorGradingPreviews({ const attempt = async () => { if (cancelled || complete || inFlight) return; - const runtime = readRuntime(iframe); + const runtime = previewRuntime(iframe); if (!runtime) return; inFlight = true; try { @@ -270,45 +349,50 @@ export function useColorGradingPreviews({ stopAnimatedPreview(); postColorGrading(next ?? gradingRef.current); if (!next || !options) return; - const runtime = readRuntime(previewIframeRef?.current); + const animate = (runtime: RuntimeColorGradingPreview, session: AnimatedSession) => { + const renderFrame = async () => { + try { + const batch = await runtime.renderPreviews( + target, + [{ id: options.animatedPreview.id, grading: toPreviewColorGrading(next) }], + { maxDimension: previewMaxDimension(), useMediaTime: true }, + ); + const image = batch?.images[0]?.dataUrl; + if (animatedRef.current !== session || !batch || !image) return; + setState((current) => ({ + ...current, + previews: { + ...current.previews, + [options.animatedPreview.kind]: { + ...current.previews[options.animatedPreview.kind], + status: "ready", + images: { + ...current.previews[options.animatedPreview.kind].images, + [options.animatedPreview.id]: image, + }, + width: batch.width, + height: batch.height, + }, + }, + })); + } catch { + // A later frame can recover after media/runtime readiness races. + } finally { + if (animatedRef.current === session) { + session.timer = window.setTimeout(() => void renderFrame(), 100); + } + } + }; + void renderFrame(); + }; + const iframe = previewIframeRef?.current; + if (playInSealedFrame(iframe, target, animatedRef, animate)) return; + const runtime = readRuntime(iframe); const stopPlayback = runtime?.startPreviewPlayback?.(target); if (!runtime || !stopPlayback) return; const session = { stopPlayback, timer: null as number | null }; animatedRef.current = session; - const renderFrame = async () => { - try { - const batch = await runtime.renderPreviews( - target, - [{ id: options.animatedPreview.id, grading: toPreviewColorGrading(next) }], - { maxDimension: previewMaxDimension(), useMediaTime: true }, - ); - const image = batch?.images[0]?.dataUrl; - if (animatedRef.current !== session || !batch || !image) return; - setState((current) => ({ - ...current, - previews: { - ...current.previews, - [options.animatedPreview.kind]: { - ...current.previews[options.animatedPreview.kind], - status: "ready", - images: { - ...current.previews[options.animatedPreview.kind].images, - [options.animatedPreview.id]: image, - }, - width: batch.width, - height: batch.height, - }, - }, - })); - } catch { - // A later frame can recover after media/runtime readiness races. - } finally { - if (animatedRef.current === session) { - session.timer = window.setTimeout(() => void renderFrame(), 100); - } - } - }; - void renderFrame(); + animate(runtime, session); }, [gradingRef, postColorGrading, previewIframeRef, stopAnimatedPreview, target], ); @@ -336,7 +420,7 @@ export function useColorGradingPreviews({ }: { grading?: NormalizedHfColorGrading; } = {}): Promise => { - const runtime = readRuntime(previewIframeRef?.current); + const runtime = previewRuntime(previewIframeRef?.current); if (!runtime) return null; const batch = await runtime.renderPreviews( target, diff --git a/scripts/package-size-budgets.json b/scripts/package-size-budgets.json index 0a4ef3906b..98df2178d3 100644 --- a/scripts/package-size-budgets.json +++ b/scripts/package-size-budgets.json @@ -21,7 +21,7 @@ }, "@hyperframes/inspector": { "packed": 880000, - "unpacked": 3820000 + "unpacked": 3830000 }, "@hyperframes/lint": { "packed": 191022,