feat(runtime): apply color grading in preview and render

packages/core/src/inline-scripts/runtimeContract.ts

@@ -17,6 +17,8 @@ export const HYPERFRAME_CONTROL_ACTIONS = [
   "seek",
   "set-muted",
   "set-playback-rate",
+  "set-color-grading",
+  "set-color-grading-compare",
   "enable-pick-mode",
   "disable-pick-mode",
 ] as const;

packages/core/src/runtime/bridge.test.ts

@@ -11,6 +11,8 @@ function createMockDeps() {
     onSetVolume: vi.fn(),
     onSetMediaOutputMuted: vi.fn(),
     onSetPlaybackRate: vi.fn(),
+    onSetColorGrading: vi.fn(),
+    onSetColorGradingCompare: vi.fn(),
     onEnablePickMode: vi.fn(),
     onDisablePickMode: vi.fn(),
   };
@@ -111,6 +113,30 @@ describe("installRuntimeControlBridge", () => {
     expect(deps.onSetPlaybackRate).toHaveBeenCalledWith(1);
   });
 
+  it("dispatches set-color-grading command with target and grading payload", () => {
+    const deps = createMockDeps();
+    const handler = installRuntimeControlBridge(deps);
+    const grading = { preset: "warm-clean", intensity: 0.7 };
+    const target = { id: "hero-video", selectorIndex: 0 };
+    handler(makeControlMessage("set-color-grading", { target, grading }));
+    expect(deps.onSetColorGrading).toHaveBeenCalledWith(
+      { id: "hero-video", hfId: null, selector: null, selectorIndex: 0 },
+      grading,
+    );
+  });
+
+  it("dispatches set-color-grading-compare command with target and compare payload", () => {
+    const deps = createMockDeps();
+    const handler = installRuntimeControlBridge(deps);
+    const compare = { enabled: true, position: 0.42 };
+    const target = { id: "hero-video", selectorIndex: 0 };
+    handler(makeControlMessage("set-color-grading-compare", { target, compare }));
+    expect(deps.onSetColorGradingCompare).toHaveBeenCalledWith(
+      { id: "hero-video", hfId: null, selector: null, selectorIndex: 0 },
+      compare,
+    );
+  });
+
   it("dispatches tick command", () => {
     const deps = createMockDeps();
     const handler = installRuntimeControlBridge(deps);

packages/core/src/runtime/bridge.ts

@@ -1,4 +1,5 @@
 import { swallow } from "./diagnostics";
+import type { HfColorGradingTarget } from "../colorGrading";
 import type { RuntimeBridgeControlMessage, RuntimeOutboundMessage } from "./types";
 
 type BridgeDeps = {
@@ -10,10 +11,39 @@ type BridgeDeps = {
   onSetVolume: (volume: number) => void;
   onSetMediaOutputMuted: (muted: boolean) => void;
   onSetPlaybackRate: (rate: number) => void;
+  onSetColorGrading: (target: HfColorGradingTarget | string | null, grading: unknown) => void;
+  onSetColorGradingCompare: (
+    target: HfColorGradingTarget | string | null,
+    compare: unknown,
+  ) => void;
   onEnablePickMode: () => void;
   onDisablePickMode: () => void;
 };
 
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+function readOptionalString(value: unknown): string | null {
+  return typeof value === "string" && value.trim() ? value : null;
+}
+
+function readOptionalIndex(value: unknown): number | null {
+  const parsed = Number(value);
+  return Number.isFinite(parsed) && parsed >= 0 ? Math.floor(parsed) : null;
+}
+
+function readColorGradingTarget(value: unknown): HfColorGradingTarget | string | null {
+  if (typeof value === "string") return value;
+  if (!isRecord(value)) return null;
+  return {
+    id: readOptionalString(value.id),
+    hfId: readOptionalString(value.hfId),
+    selector: readOptionalString(value.selector),
+    selectorIndex: readOptionalIndex(value.selectorIndex),
+  };
+}
+
 export function postRuntimeMessage(payload: RuntimeOutboundMessage): void {
   try {
     window.parent.postMessage(payload, "*");
@@ -60,6 +90,19 @@ export function installRuntimeControlBridge(deps: BridgeDeps): (event: MessageEv
       deps.onSetPlaybackRate(Number(data.playbackRate ?? 1));
       return;
     }
+    if (action === "set-color-grading") {
+      const payload = isRecord(data) ? data : {};
+      deps.onSetColorGrading(readColorGradingTarget(payload.target), payload.grading ?? null);
+      return;
+    }
+    if (action === "set-color-grading-compare") {
+      const payload = isRecord(data) ? data : {};
+      deps.onSetColorGradingCompare(
+        readColorGradingTarget(payload.target),
+        payload.compare ?? null,
+      );
+      return;
+    }
     if (action === "enable-pick-mode") {
       deps.onEnablePickMode();
       return;

packages/core/src/runtime/colorGrading.test.ts

@@ -0,0 +1,308 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import { HF_COLOR_GRADING_ATTR, serializeHfColorGrading } from "../colorGrading";
+import { createColorGradingRuntime, type RuntimeColorGradingApi } from "./colorGrading";
+
+let lastUniform1f: ReturnType<typeof vi.fn> | null = null;
+let lastUniform3f: ReturnType<typeof vi.fn> | null = null;
+
+const IDENTITY_2 = `
+LUT_3D_SIZE 2
+0 0 0
+1 0 0
+0 1 0
+1 1 0
+0 0 1
+1 0 1
+0 1 1
+1 1 1
+`;
+
+function createMockWebGl(): WebGLRenderingContext {
+  const shader = {};
+  const program = {};
+  const texture = {};
+  const buffer = {};
+  const uniform1f = vi.fn();
+  const uniform3f = vi.fn();
+  lastUniform1f = uniform1f;
+  lastUniform3f = uniform3f;
+  return {
+    VERTEX_SHADER: 0x8b31,
+    FRAGMENT_SHADER: 0x8b30,
+    COMPILE_STATUS: 0x8b81,
+    LINK_STATUS: 0x8b82,
+    TEXTURE_2D: 0x0de1,
+    TEXTURE_WRAP_S: 0x2802,
+    TEXTURE_WRAP_T: 0x2803,
+    TEXTURE_MIN_FILTER: 0x2801,
+    TEXTURE_MAG_FILTER: 0x2800,
+    CLAMP_TO_EDGE: 0x812f,
+    LINEAR: 0x2601,
+    NEAREST: 0x2600,
+    RGBA: 0x1908,
+    UNSIGNED_BYTE: 0x1401,
+    ARRAY_BUFFER: 0x8892,
+    STATIC_DRAW: 0x88e4,
+    TEXTURE0: 0x84c0,
+    TEXTURE1: 0x84c1,
+    FLOAT: 0x1406,
+    TRIANGLE_STRIP: 0x0005,
+    UNPACK_FLIP_Y_WEBGL: 0x9240,
+    createShader: vi.fn(() => shader),
+    shaderSource: vi.fn(),
+    compileShader: vi.fn(),
+    getShaderParameter: vi.fn(() => true),
+    getShaderInfoLog: vi.fn(() => ""),
+    deleteShader: vi.fn(),
+    createProgram: vi.fn(() => program),
+    attachShader: vi.fn(),
+    linkProgram: vi.fn(),
+    getProgramParameter: vi.fn(() => true),
+    getProgramInfoLog: vi.fn(() => ""),
+    deleteProgram: vi.fn(),
+    createTexture: vi.fn(() => texture),
+    bindTexture: vi.fn(),
+    texParameteri: vi.fn(),
+    texImage2D: vi.fn(),
+    createBuffer: vi.fn(() => buffer),
+    bindBuffer: vi.fn(),
+    bufferData: vi.fn(),
+    getAttribLocation: vi.fn(() => 0),
+    getUniformLocation: vi.fn((_program, name: string) => name),
+    viewport: vi.fn(),
+    useProgram: vi.fn(),
+    activeTexture: vi.fn(),
+    pixelStorei: vi.fn(),
+    uniform1i: vi.fn(),
+    uniform2f: vi.fn(),
+    uniform1f,
+    uniform3f,
+    enableVertexAttribArray: vi.fn(),
+    vertexAttribPointer: vi.fn(),
+    drawArrays: vi.fn(),
+    deleteTexture: vi.fn(),
+  } as unknown as WebGLRenderingContext;
+}
+
+function makeDrawableVideo(): HTMLVideoElement {
+  const video = document.createElement("video");
+  video.id = "hero-video";
+  video.setAttribute(HF_COLOR_GRADING_ATTR, serializeHfColorGrading({ adjust: { exposure: 0.5 } }));
+  Object.defineProperty(video, "readyState", {
+    value: HTMLMediaElement.HAVE_CURRENT_DATA,
+    configurable: true,
+  });
+  Object.defineProperty(video, "videoWidth", { value: 640, configurable: true });
+  Object.defineProperty(video, "videoHeight", { value: 360, configurable: true });
+  Object.defineProperty(video, "offsetWidth", { value: 640, configurable: true });
+  Object.defineProperty(video, "offsetHeight", { value: 360, configurable: true });
+  Object.defineProperty(video, "offsetLeft", { value: 0, configurable: true });
+  Object.defineProperty(video, "offsetTop", { value: 0, configurable: true });
+  video.getBoundingClientRect = () =>
+    ({
+      x: 0,
+      y: 0,
+      left: 0,
+      top: 0,
+      right: 640,
+      bottom: 360,
+      width: 640,
+      height: 360,
+      toJSON: () => ({}),
+    }) as DOMRect;
+  return video;
+}
+
+function stubCubeLutFetch(): ReturnType<typeof vi.fn> {
+  const fetchMock = vi.fn(() =>
+    Promise.resolve({
+      ok: true,
+      status: 200,
+      text: () => Promise.resolve(IDENTITY_2),
+    }),
+  );
+  vi.stubGlobal("fetch", fetchMock);
+  return fetchMock;
+}
+
+describe("createColorGradingRuntime", () => {
+  let getContextSpy: ReturnType<typeof vi.spyOn>;
+  let runtime: RuntimeColorGradingApi | null = null;
+
+  beforeEach(() => {
+    document.body.innerHTML = "";
+    lastUniform1f = null;
+    lastUniform3f = null;
+    getContextSpy = vi
+      .spyOn(HTMLCanvasElement.prototype, "getContext")
+      .mockImplementation((type: string) =>
+        type === "webgl" ? createMockWebGl() : null,
+      ) as ReturnType<typeof vi.spyOn>;
+  });
+
+  afterEach(() => {
+    runtime?.destroy();
+    runtime = null;
+    vi.unstubAllGlobals();
+    getContextSpy.mockRestore();
+    delete window.__hfVariables;
+    delete window.__hfVariablesByComp;
+    document.head.innerHTML = "";
+    document.body.innerHTML = "";
+  });
+
+  function startRuntimeWithVideo(video = makeDrawableVideo()): {
+    video: HTMLVideoElement;
+    canvas: HTMLCanvasElement;
+  } {
+    document.body.appendChild(video);
+    runtime = createColorGradingRuntime();
+    const canvas = document.querySelector<HTMLCanvasElement>("[data-hf-color-grading-canvas]");
+    if (!canvas) throw new Error("Expected color grading canvas");
+    return { video, canvas };
+  }
+
+  async function flushLutLoad(): Promise<void> {
+    await Promise.resolve();
+    await Promise.resolve();
+    await new Promise((resolve) => window.setTimeout(resolve, 0));
+    runtime?.redraw();
+  }
+
+  it("re-hides source media after timeline visibility sync", () => {
+    const { video, canvas } = startRuntimeWithVideo();
+
+    expect(video.style.getPropertyValue("visibility")).toBe("");
+    expect(video.style.getPropertyValue("opacity")).toBe("0");
+    expect(video.style.getPropertyPriority("opacity")).toBe("important");
+    expect(video.hasAttribute("data-hf-color-grading-source-hidden")).toBe(true);
+    expect(canvas?.style.visibility).toBe("visible");
+    expect(canvas?.style.opacity).toBe("1");
+
+    video.style.visibility = "visible";
+    runtime.setSourceVisibility(video, true);
+    runtime.redraw();
+
+    expect(video.style.getPropertyValue("visibility")).toBe("visible");
+    expect(video.style.getPropertyValue("opacity")).toBe("0");
+    expect(video.style.getPropertyPriority("opacity")).toBe("important");
+    expect(canvas?.style.visibility).toBe("visible");
+
+    video.style.visibility = "hidden";
+    runtime.setSourceVisibility(video, false);
+    runtime.redraw();
+
+    expect(video.style.getPropertyValue("visibility")).toBe("hidden");
+    expect(video.style.getPropertyValue("opacity")).toBe("0");
+    expect(video.style.getPropertyPriority("opacity")).toBe("important");
+    expect(canvas?.style.visibility).toBe("hidden");
+  });
+
+  it("resolves grading values from the nearest sub-composition variable scope", () => {
+    window.__hfVariables = {
+      exposure: -0.25,
+    };
+    window.__hfVariablesByComp = {
+      card__hf1: {
+        exposure: 0.75,
+      },
+    };
+    const host = document.createElement("div");
+    host.setAttribute("data-composition-id", "card__hf1");
+    const video = makeDrawableVideo();
+    video.id = "first-video";
+    video.setAttribute(
+      HF_COLOR_GRADING_ATTR,
+      JSON.stringify({ adjust: { exposure: "$exposure" } }),
+    );
+    host.appendChild(video);
+    document.body.appendChild(host);
+
+    runtime = createColorGradingRuntime();
+
+    if (!lastUniform1f) throw new Error("Expected WebGL uniform calls");
+    expect(lastUniform1f).toHaveBeenCalledWith("u_exposure", 0.75);
+  });
+
+  it("falls back to top-level variables for root media color grading", () => {
+    window.__hfVariables = {
+      exposure: 0.35,
+    };
+    const video = makeDrawableVideo();
+    video.setAttribute(
+      HF_COLOR_GRADING_ATTR,
+      JSON.stringify({ adjust: { exposure: "${exposure}" } }),
+    );
+    document.body.appendChild(video);
+
+    runtime = createColorGradingRuntime();
+
+    if (!lastUniform1f) throw new Error("Expected WebGL uniform calls");
+    expect(lastUniform1f).toHaveBeenCalledWith("u_exposure", 0.35);
+  });
+
+  it("keeps the last shader frame visible while a video seek is waiting for a drawable frame", () => {
+    const { video, canvas } = startRuntimeWithVideo();
+
+    expect(canvas.style.display).toBe("block");
+
+    Object.defineProperty(video, "readyState", {
+      value: HTMLMediaElement.HAVE_METADATA,
+      configurable: true,
+    });
+
+    runtime.redraw();
+
+    expect(canvas.style.display).toBe("block");
+    expect(video.style.getPropertyValue("opacity")).toBe("0");
+    expect(video.style.getPropertyPriority("opacity")).toBe("important");
+  });
+
+  it("updates before-after compare uniforms without changing the source grading", () => {
+    const video = makeDrawableVideo();
+    document.body.appendChild(video);
+
+    runtime = createColorGradingRuntime();
+    const updated = runtime.setCompare("#hero-video", {
+      enabled: true,
+      position: 0.25,
+      lineWidth: 4,
+    });
+
+    if (!lastUniform1f) throw new Error("Expected WebGL uniform calls");
+    expect(updated).toBe(true);
+    expect(lastUniform1f).toHaveBeenCalledWith("u_compareEnabled", 1);
+    expect(lastUniform1f).toHaveBeenCalledWith("u_comparePosition", 0.25);
+    expect(lastUniform1f).toHaveBeenCalledWith("u_compareLineWidth", 4);
+    expect(video.getAttribute(HF_COLOR_GRADING_ATTR)).toBe(
+      serializeHfColorGrading({ adjust: { exposure: 0.5 } }),
+    );
+  });
+
+  it("loads cube LUTs and enables LUT uniforms", async () => {
+    const fetchMock = stubCubeLutFetch();
+    const origin = window.location.origin;
+    document.head.innerHTML = `<base href="${origin}/api/projects/demo/preview/">`;
+    const video = makeDrawableVideo();
+    video.setAttribute(
+      HF_COLOR_GRADING_ATTR,
+      serializeHfColorGrading({ lut: { src: "assets/luts/identity.cube", intensity: 0.4 } }),
+    );
+    document.body.appendChild(video);
+
+    runtime = createColorGradingRuntime();
+    await flushLutLoad();
+
+    expect(fetchMock).toHaveBeenCalledWith(
+      `${origin}/api/projects/demo/preview/assets/luts/identity.cube`,
+      { credentials: "same-origin" },
+    );
+    if (!lastUniform1f || !lastUniform3f) throw new Error("Expected WebGL uniform calls");
+    expect(lastUniform1f).toHaveBeenCalledWith("u_lutEnabled", 1);
+    expect(lastUniform1f).toHaveBeenCalledWith("u_lutSize", 2);
+    expect(lastUniform1f).toHaveBeenCalledWith("u_lutIntensity", 0.4);
+    expect(lastUniform3f).toHaveBeenCalledWith("u_lutDomainMin", 0, 0, 0);
+    expect(lastUniform3f).toHaveBeenCalledWith("u_lutDomainMax", 1, 1, 1);
+    expect(runtime.getStatus("#hero-video").message).toBe("Shader + LUT active");
+  });
+});

packages/core/src/runtime/init.ts

@@ -23,6 +23,7 @@ import { createRuntimeStartTimeResolver } from "./startResolver";
 import { createClipTree } from "./clipTree";
 import { loadExternalCompositions, loadInlineTemplateCompositions } from "./compositionLoader";
 import { applyCaptionOverrides } from "./captionOverrides";
+import { createColorGradingRuntime, type RuntimeColorGradingApi } from "./colorGrading";
 import { TransportClock } from "./clock";
 import { WebAudioTransport } from "./webAudioTransport";
 import { quantizeTimeToFrame } from "../inline-scripts/parityContract";
@@ -36,6 +37,7 @@ const AUTHORED_END_ATTR = "data-hf-authored-end";
 
 export function initSandboxRuntimeModular(): void {
   const state = createRuntimeState();
+  let colorGradingRuntime: RuntimeColorGradingApi | null = null;
   let runtimeErrorListener: ((event: ErrorEvent) => void) | null = null;
   let runtimeUnhandledRejectionListener: ((event: PromiseRejectionEvent) => void) | null = null;
   const runtimeCleanupCallbacks: Array<() => void> = [];
@@ -1534,6 +1536,9 @@ export function initSandboxRuntimeModular(): void {
         }
       }
       rawNode.style.visibility = isVisibleNow ? "visible" : "hidden";
+      if (rawNode instanceof HTMLVideoElement || rawNode instanceof HTMLImageElement) {
+        colorGradingRuntime?.setSourceVisibility(rawNode, isVisibleNow);
+      }
     }
   };
 
@@ -1678,6 +1683,13 @@ export function initSandboxRuntimeModular(): void {
   });
   picker.installPickerApi();
 
+  const colorGrading = createColorGradingRuntime();
+  colorGradingRuntime = colorGrading;
+  registerRuntimeCleanup(() => {
+    colorGrading.destroy();
+    colorGradingRuntime = null;
+  });
+
   const applyPlaybackRate = (nextRate: number) => {
     const parsed = Number(nextRate);
     if (!Number.isFinite(parsed) || parsed <= 0) {
@@ -1735,7 +1747,9 @@ export function initSandboxRuntimeModular(): void {
     },
     onDeterministicPause: () => runAdapters("pause"),
     onDeterministicPlay: () => runAdapters("play"),
-    onRenderFrameSeek: () => {},
+    onRenderFrameSeek: () => {
+      colorGrading.redraw();
+    },
     onShowNativeVideos: () => {},
     getSafeDuration: () => getSafeTimelineDurationSeconds(state.capturedTimeline, 0),
   });
@@ -1801,6 +1815,12 @@ export function initSandboxRuntimeModular(): void {
       if (state.transportClock) state.transportClock.setRate(state.playbackRate);
       applyWebAudioRate();
     },
+    onSetColorGrading: (target, grading) => {
+      colorGrading.setGrading(target, grading);
+    },
+    onSetColorGradingCompare: (target, compare) => {
+      colorGrading.setCompare(target, compare);
+    },
     onTick: () => {
       if (state.tornDown || !clock.isPlaying()) return;
       const t = clock.now();
@@ -2288,6 +2308,7 @@ export function initSandboxRuntimeModular(): void {
     if (webAudioReady) scheduleWebAudioForActiveClips();
     runAdapters("play");
     syncMediaForCurrentState();
+    colorGrading.redraw();
     postState(true);
   };
 
@@ -2304,6 +2325,7 @@ export function initSandboxRuntimeModular(): void {
     if (tl) tl.pause();
     runAdapters("pause");
     syncMediaForCurrentState();
+    colorGrading.redraw();
     postState(true);
   };
 
@@ -2325,6 +2347,7 @@ export function initSandboxRuntimeModular(): void {
     seekTimelineAndAdapters(state.currentTime);
     runAdapters("pause");
     syncMediaForCurrentState();
+    colorGrading.redraw();
     postState(true);
   };
 
@@ -2340,6 +2363,7 @@ export function initSandboxRuntimeModular(): void {
     state.mediaForceSyncNextTick = true;
     seekTimelineAndAdapters(state.currentTime, { activateChildren: true });
     syncMediaForCurrentState();
+    colorGrading.redraw();
     postState(true);
   };
 

packages/core/src/runtime/picker.ts

@@ -17,6 +17,7 @@ const PICKER_BLOCK_SELECTOR = [
   "[data-hyperframes-picker-block]",
   "[data-hyper-shader-loading]",
 ].join(",");
+const COLOR_GRADING_SOURCE_HIDDEN_ATTR = "data-hf-color-grading-source-hidden";
 
 export type PickerModule = {
   enablePickMode: () => void;
@@ -67,7 +68,12 @@ export function createPickerModule(deps: PickerModuleDeps): PickerModule {
       if (computed.display === "none" || computed.visibility === "hidden") return true;
       if (computed.pointerEvents === "none") return true;
       const opacity = Number.parseFloat(computed.opacity);
-      if (Number.isFinite(opacity) && opacity <= 0.01) return true;
+      if (
+        Number.isFinite(opacity) &&
+        opacity <= 0.01 &&
+        !current.hasAttribute(COLOR_GRADING_SOURCE_HIDDEN_ATTR)
+      )
+        return true;
       current = current.parentElement;
     }
     return false;

packages/core/src/runtime/types.ts

@@ -1,3 +1,5 @@
+import type { HfColorGradingTarget } from "../colorGrading";
+
 export type RuntimeJson =
   | string
   | number
@@ -24,6 +26,9 @@ export type RuntimeBridgeControlMessage = {
   muted?: boolean;
   volume?: number;
   playbackRate?: number;
+  target?: HfColorGradingTarget | string | null;
+  grading?: RuntimeJson;
+  compare?: RuntimeJson;
   seekMode?: "drag" | "commit";
 };
 

packages/core/src/runtime/window.d.ts

@@ -1,4 +1,5 @@
 import type { RuntimeTimelineMessage, RuntimeTimelineLike } from "./types";
+import type { RuntimeColorGradingApi } from "./colorGrading";
 import type { HyperframePickerApi } from "../inline-scripts/pickerApi";
 import type { PlayerAPI } from "../core.types";
 import type { ClipTree } from "./clipTree";
@@ -31,6 +32,10 @@ declare global {
     __player?: PlayerAPI;
     __clipManifest?: RuntimeTimelineMessage;
     __clipTree?: ClipTree;
+    __hf?: {
+      colorGrading?: RuntimeColorGradingApi;
+      onSwallowed?: (label: string, err: unknown) => void;
+    };
     __playerReady?: boolean;
     __renderReady?: boolean;
     __hfRuntimeTeardown?: (() => void) | null;

packages/core/src/studio-api/routes/preview.ts

@@ -326,7 +326,7 @@ export function registerPreviewRoutes(api: Hono, adapter: StudioApiAdapter): voi
       return c.text("not found", 404);
     }
     const contentType = getMimeType(subPath);
-    const isText = /\.(html|css|js|json|svg|txt|md)$/i.test(subPath);
+    const isText = /\.(html|css|js|json|svg|txt|md|cube)$/i.test(subPath);
 
     const etag = `"${stat.mtimeMs.toString(36)}-${stat.size.toString(36)}"`;
     const cacheHeaders: Record<string, string> = isText