Merge pull request #2752 from heygen-com/feat/media-treatment-runtime

feat(runtime): render media treatments deterministically
This commit is contained in:
Ular Kimsanov
2026-07-24 12:05:14 -07:00
committed by GitHub
13 changed files with 2994 additions and 309 deletions
+27
View File
@@ -13,6 +13,7 @@ import {
normalizeHfColorGradingWithVariables,
serializeHfColorGrading,
} from "./colorGrading";
import { lintHyperframeHtml } from "./lint";
describe("color grading", () => {
it("derives grade and effect preset views from their actual payloads", () => {
@@ -28,6 +29,32 @@ describe("color grading", () => {
expect(HF_COLOR_GRADING_PRESETS).toHaveLength(18);
});
it("keeps every canonical grading key accepted by lint", async () => {
const grading = normalizeHfColorGrading("neutral");
expect(grading).not.toBeNull();
const html = (attribute: string) => `
<html><body>
<div id="root" data-composition-id="c1" data-start="0" data-width="1920" data-height="1080" data-duration="1">
<img class="clip" data-start="0" data-duration="1" src="media.jpg" data-color-grading='${attribute}'>
</div>
<script>window.__timelines = {};</script>
</body></html>
`;
const valid = await lintHyperframeHtml(html(serializeHfColorGrading(grading)));
expect(valid.findings.filter((finding) => finding.severity === "error")).toEqual([]);
const invalid = await lintHyperframeHtml(html('{"effects":{"notARealEffect":1}}'));
expect(invalid.findings).toEqual(
expect.arrayContaining([
expect.objectContaining({
code: "color_grading_invalid_structure",
severity: "error",
}),
]),
);
});
it("parses preset shorthand", () => {
const grading = normalizeHfColorGrading("warm-daylight");
expect(grading?.preset).toBe("warm-daylight");
+748 -38
View File
@@ -11,6 +11,7 @@ let lastUniform1f: ReturnType<typeof vi.fn> | null = null;
let lastUniform3f: ReturnType<typeof vi.fn> | null = null;
let lastShaderSources: string[] = [];
let texImage2DCalls: unknown[][] = [];
let loseContextCalls = 0;
const IDENTITY_2 = `
LUT_3D_SIZE 2
@@ -24,7 +25,9 @@ LUT_3D_SIZE 2
1 1 1
`;
function createMockWebGl(options: { failMediaUpload?: boolean } = {}): WebGLRenderingContext {
function createMockWebGl(
options: { failMediaUpload?: boolean; halfFloatSupported?: boolean } = {},
): WebGLRenderingContext {
const shader = {};
const program = {};
const texture = {};
@@ -54,6 +57,7 @@ function createMockWebGl(options: { failMediaUpload?: boolean } = {}): WebGLRend
TEXTURE1: 0x84c1,
TEXTURE2: 0x84c2,
TEXTURE3: 0x84c3,
TEXTURE4: 0x84c4,
FLOAT: 0x1406,
TRIANGLE_STRIP: 0x0005,
UNPACK_FLIP_Y_WEBGL: 0x9240,
@@ -88,6 +92,14 @@ function createMockWebGl(options: { failMediaUpload?: boolean } = {}): WebGLRend
framebufferTexture2D: vi.fn(),
checkFramebufferStatus: vi.fn(() => 0x8cd5),
deleteFramebuffer: vi.fn(),
getExtension: vi.fn((name: string) => {
if (name === "WEBGL_lose_context") {
return { loseContext: () => (loseContextCalls += 1) };
}
if (options.halfFloatSupported === false) return null;
if (name === "OES_texture_half_float") return { HALF_FLOAT_OES: 0x8d61 };
return name === "EXT_color_buffer_half_float" ? {} : null;
}),
createBuffer: vi.fn(() => buffer),
bindBuffer: vi.fn(),
bufferData: vi.fn(),
@@ -138,6 +150,21 @@ function makeDrawableVideo(): HTMLVideoElement {
return video;
}
function makeDrawableImage(): HTMLImageElement {
const image = document.createElement("img");
image.id = "hero-image";
image.setAttribute(HF_COLOR_GRADING_ATTR, serializeHfColorGrading({ effects: { blur: 0 } }));
image.style.setProperty("--hf-color-grading-blur", "0");
Object.defineProperty(image, "complete", { value: true, configurable: true });
Object.defineProperty(image, "naturalWidth", { value: 640, configurable: true });
Object.defineProperty(image, "naturalHeight", { value: 360, configurable: true });
Object.defineProperty(image, "offsetWidth", { value: 640, configurable: true });
Object.defineProperty(image, "offsetHeight", { value: 360, configurable: true });
image.getBoundingClientRect = () =>
({ width: 640, height: 360, left: 0, top: 0, right: 640, bottom: 360 }) as DOMRect;
return image;
}
function stubCubeLutFetch(text = IDENTITY_2): ReturnType<typeof vi.fn> {
const fetchMock = vi.fn(() =>
Promise.resolve({
@@ -160,6 +187,7 @@ describe("createColorGradingRuntime", () => {
lastUniform3f = null;
lastShaderSources = [];
texImage2DCalls = [];
loseContextCalls = 0;
getContextSpy = vi
.spyOn(HTMLCanvasElement.prototype, "getContext")
.mockImplementation((type: string) =>
@@ -174,6 +202,7 @@ describe("createColorGradingRuntime", () => {
getContextSpy.mockRestore();
delete window.__hfVariables;
delete window.__hfVariablesByComp;
delete window.__player;
document.head.innerHTML = "";
document.body.innerHTML = "";
});
@@ -189,6 +218,32 @@ describe("createColorGradingRuntime", () => {
return { video, canvas };
}
it.each([
{ pixelRatio: 1, width: 640, height: 360 },
{ pixelRatio: 2, width: 1280, height: 720 },
])(
"matches the WebGL drawing buffer to the displayed media at DPR $pixelRatio",
({ pixelRatio, width, height }) => {
vi.stubGlobal("devicePixelRatio", pixelRatio);
const { canvas } = startRuntimeWithVideo();
expect(canvas.style.width).toBe("640px");
expect(canvas.style.height).toBe("360px");
expect(canvas.width).toBe(width);
expect(canvas.height).toBe(height);
},
);
it("uses the default non-preserved drawing buffer outside capture instrumentation", () => {
startRuntimeWithVideo();
expect(getContextSpy).toHaveBeenCalledWith("webgl", {
alpha: true,
premultipliedAlpha: false,
});
});
async function flushLutLoad(): Promise<void> {
await Promise.resolve();
await Promise.resolve();
@@ -250,7 +305,7 @@ describe("createColorGradingRuntime", () => {
expect(texImage2DCalls.length).toBe(drawsBefore);
});
it("re-hides source media after timeline visibility sync", () => {
it("releases inactive attribute grading and recreates it when visible", () => {
const { video, canvas } = startRuntimeWithVideo();
expect(canvas.id).toBe("__hf_color_grading_hero-video");
@@ -261,23 +316,177 @@ describe("createColorGradingRuntime", () => {
expect(canvas?.style.visibility).toBe("visible");
expect(canvas?.style.opacity).toBe("1");
video.style.visibility = "visible";
runtime.setSourceVisibility(video, true);
runtime.redraw();
expect(video.style.getPropertyValue("visibility")).toBe("visible");
expect(video.style.getPropertyValue("opacity")).toBe("0");
expect(video.style.getPropertyPriority("opacity")).toBe("important");
expect(canvas?.style.visibility).toBe("visible");
video.style.visibility = "hidden";
runtime.setSourceVisibility(video, false);
runtime.redraw();
expect(runtime.setSourceVisibility(video, false)).toBe(true);
expect(video.style.getPropertyValue("visibility")).toBe("hidden");
expect(video.style.getPropertyValue("opacity")).toBe("");
expect(video.hasAttribute("data-hf-color-grading-source-hidden")).toBe(false);
expect(canvas.isConnected).toBe(false);
video.style.visibility = "visible";
expect(runtime.setSourceVisibility(video, true)).toBe(true);
const recreated = document.querySelector<HTMLCanvasElement>("[data-hf-color-grading-canvas]");
expect(recreated).not.toBeNull();
expect(recreated).toBe(canvas);
expect(getContextSpy).toHaveBeenCalledTimes(1);
expect(video.style.getPropertyValue("opacity")).toBe("0");
expect(video.style.getPropertyPriority("opacity")).toBe("important");
expect(canvas?.style.visibility).toBe("hidden");
});
it("defers WebGL setup for hidden attributed media until it becomes visible", () => {
const video = makeDrawableVideo();
video.style.display = "none";
document.body.appendChild(video);
runtime = createColorGradingRuntime();
expect(getContextSpy).not.toHaveBeenCalled();
expect(document.querySelector("[data-hf-color-grading-canvas]")).toBeNull();
expect(runtime.getStatus(video)).toEqual({
state: "pending",
message: "Waiting for visible media",
});
video.style.display = "block";
expect(runtime.setSourceVisibility(video, true)).toBe(true);
expect(getContextSpy).toHaveBeenCalledTimes(1);
expect(document.querySelector("[data-hf-color-grading-canvas]")).not.toBeNull();
});
it("renders exact preset previews for ungraded media without replacing the source", async () => {
const video = makeDrawableVideo();
video.removeAttribute(HF_COLOR_GRADING_ATTR);
document.body.appendChild(video);
const toDataUrl = vi
.spyOn(HTMLCanvasElement.prototype, "toDataURL")
.mockReturnValue("data:image/png;base64,preview");
runtime = createColorGradingRuntime();
const batch = await runtime.renderPreviews(
"#hero-video",
[
{ id: "clean", grading: "clean-studio" },
{ id: "warm", grading: "warm-daylight" },
],
{ maxDimension: 160 },
);
expect(batch).toMatchObject({
width: 160,
height: 90,
images: [
{ id: "clean", dataUrl: "data:image/png;base64,preview" },
{ id: "warm", dataUrl: "data:image/png;base64,preview" },
],
});
expect(getContextSpy).toHaveBeenCalledTimes(1);
expect(document.querySelector("[data-hf-color-grading-canvas]")).toBeNull();
expect(video.style.opacity).toBe("");
await runtime.renderPreviews("#hero-video", [{ id: "mono", grading: "mono-clean" }]);
expect(getContextSpy).toHaveBeenCalledTimes(1);
const retinaBatch = await runtime.renderPreviews(
"#hero-video",
[{ id: "retina", grading: "neutral" }],
{ maxDimension: 400 },
);
expect(retinaBatch).toMatchObject({ width: 320, height: 180 });
toDataUrl.mockRestore();
});
it("plays only a selected video for preview and restores its media state", () => {
const video = makeDrawableVideo();
let paused = true;
Object.defineProperty(video, "paused", { configurable: true, get: () => paused });
Object.defineProperty(video, "currentTime", {
configurable: true,
value: 3.25,
writable: true,
});
Object.defineProperty(video, "duration", { configurable: true, value: 10 });
const play = vi.spyOn(video, "play").mockImplementation(() => {
paused = false;
return Promise.resolve();
});
const pause = vi.spyOn(video, "pause").mockImplementation(() => {
paused = true;
});
video.loop = false;
video.muted = false;
document.body.appendChild(video);
runtime = createColorGradingRuntime();
const stop = runtime.startPreviewPlayback("#hero-video");
expect(stop).not.toBeNull();
expect(play).toHaveBeenCalledTimes(1);
expect(video.loop).toBe(true);
expect(video.muted).toBe(true);
video.currentTime = 6;
stop?.();
expect(pause).toHaveBeenCalledTimes(1);
expect(video.currentTime).toBe(3.25);
expect(video.loop).toBe(false);
expect(video.muted).toBe(false);
});
it("does not start preview playback for an image", () => {
const image = makeDrawableImage();
document.body.appendChild(image);
runtime = createColorGradingRuntime();
expect(runtime.startPreviewPlayback("#hero-image")).toBeNull();
});
it("uses selected-video time for isolated animated preview frames", async () => {
const video = makeDrawableVideo();
Object.defineProperty(video, "currentTime", { configurable: true, value: 6.5 });
document.body.appendChild(video);
window.__player = { getTime: () => 1.25 };
vi.spyOn(HTMLCanvasElement.prototype, "toDataURL").mockReturnValue(
"data:image/png;base64,preview",
);
runtime = createColorGradingRuntime();
await runtime.renderPreviews("#hero-video", [{ id: "vhs", grading: { effects: { vhs: 1 } } }], {
maxDimension: 160,
useMediaTime: true,
});
expect(lastUniform1f?.mock.calls).toContainEqual(["u_effectTime", 6.5]);
});
it("uses the LUT cache and canonical multipass renderer in preview batches", async () => {
const video = makeDrawableVideo();
video.removeAttribute(HF_COLOR_GRADING_ATTR);
document.body.appendChild(video);
const fetchMock = stubCubeLutFetch();
const toDataUrl = vi
.spyOn(HTMLCanvasElement.prototype, "toDataURL")
.mockReturnValue("data:image/png;base64,lut-preview");
runtime = createColorGradingRuntime();
const batch = await runtime.renderPreviews("#hero-video", [
{
id: "lut",
grading: { preset: "warm-daylight", lut: { src: "/looks/test.cube", intensity: 0.6 } },
},
{ id: "blur", grading: { effects: { blur: 0.5 } } },
{ id: "bloom", grading: { effects: { bloom: 0.5, bloomRadius: 8 } } },
{ id: "kuwahara", grading: { effects: { kuwahara: 1, kuwaharaRadius: 0.25 } } },
]);
expect(fetchMock).toHaveBeenCalledTimes(1);
expect(batch?.images).toEqual([
{ id: "lut", dataUrl: "data:image/png;base64,lut-preview" },
{ id: "blur", dataUrl: "data:image/png;base64,lut-preview" },
{ id: "bloom", dataUrl: "data:image/png;base64,lut-preview" },
{ id: "kuwahara", dataUrl: "data:image/png;base64,lut-preview" },
]);
toDataUrl.mockRestore();
});
it("resolves grading values from the nearest sub-composition variable scope", () => {
@@ -323,6 +532,151 @@ describe("createColorGradingRuntime", () => {
expect(lastUniform1f).toHaveBeenCalledWith("u_exposure", 0.35);
});
it("samples seek-derived grading values from inline CSS properties on every redraw", () => {
const video = makeDrawableVideo();
video.setAttribute(
HF_COLOR_GRADING_ATTR,
serializeHfColorGrading({
adjust: { exposure: 0.5 },
effects: {
blur: 0.2,
bloom: 0.2,
kuwahara: 0.2,
pixelate: 0.3,
ascii: 0.4,
dither: 0.5,
},
lut: { src: "assets/luts/test.cube", intensity: 0.4 },
}),
);
video.style.setProperty("--hf-color-grading-intensity", "0.25");
video.style.setProperty("--hf-color-grading-lut-intensity", "0.35");
video.style.setProperty("--hf-color-grading-exposure", "-0.15");
video.style.setProperty("--hf-color-grading-blur", "0.45");
video.style.setProperty("--hf-color-grading-bloom", "0.45");
video.style.setProperty("--hf-color-grading-kuwahara", "0.45");
video.style.setProperty("--hf-color-grading-pixelate", "0.55");
video.style.setProperty("--hf-color-grading-ascii", "0.65");
video.style.setProperty("--hf-color-grading-dither", "0.75");
stubCubeLutFetch();
startRuntimeWithVideo(video);
if (!lastUniform1f) throw new Error("Expected WebGL uniform calls");
expect(lastUniform1f).toHaveBeenCalledWith("u_intensity", 0.25);
expect(lastUniform1f).toHaveBeenCalledWith("u_lutIntensity", 0.35);
expect(lastUniform1f).toHaveBeenCalledWith("u_exposure", -0.15);
expect(lastUniform1f).toHaveBeenCalledWith("u_blur", 0.45);
expect(lastUniform1f).toHaveBeenCalledWith("u_bloom", 0.45);
expect(lastUniform1f).toHaveBeenCalledWith("u_kuwahara", 0.45);
expect(lastUniform1f).toHaveBeenCalledWith("u_pixelate", 0.55);
expect(lastUniform1f).toHaveBeenCalledWith("u_ascii", 0.65);
expect(lastUniform1f).toHaveBeenCalledWith("u_dither", 0.75);
video.style.setProperty("--hf-color-grading-intensity", "0.75");
video.style.setProperty("--hf-color-grading-lut-intensity", "0.65");
video.style.setProperty("--hf-color-grading-exposure", "0.15");
video.style.setProperty("--hf-color-grading-blur", "0.15");
video.style.setProperty("--hf-color-grading-bloom", "0.15");
video.style.setProperty("--hf-color-grading-kuwahara", "0.15");
video.style.setProperty("--hf-color-grading-pixelate", "0.05");
video.style.setProperty("--hf-color-grading-ascii", "0.15");
video.style.setProperty("--hf-color-grading-dither", "0.25");
lastUniform1f.mockClear();
runtime?.redraw();
expect(lastUniform1f).toHaveBeenCalledWith("u_intensity", 0.75);
expect(lastUniform1f).toHaveBeenCalledWith("u_lutIntensity", 0.65);
expect(lastUniform1f).toHaveBeenCalledWith("u_exposure", 0.15);
expect(lastUniform1f).toHaveBeenCalledWith("u_blur", 0.15);
expect(lastUniform1f).toHaveBeenCalledWith("u_bloom", 0.15);
expect(lastUniform1f).toHaveBeenCalledWith("u_kuwahara", 0.15);
expect(lastUniform1f).toHaveBeenCalledWith("u_pixelate", 0.05);
expect(lastUniform1f).toHaveBeenCalledWith("u_ascii", 0.15);
expect(lastUniform1f).toHaveBeenCalledWith("u_dither", 0.25);
video.style.setProperty("--hf-color-grading-intensity", "invalid");
video.style.setProperty("--hf-color-grading-lut-intensity", "2");
video.style.setProperty("--hf-color-grading-exposure", "-3");
video.style.setProperty("--hf-color-grading-blur", "-1");
video.style.setProperty("--hf-color-grading-bloom", "invalid");
video.style.setProperty("--hf-color-grading-kuwahara", "invalid");
video.style.setProperty("--hf-color-grading-pixelate", "invalid");
video.style.setProperty("--hf-color-grading-ascii", "-1");
video.style.setProperty("--hf-color-grading-dither", "invalid");
lastUniform1f.mockClear();
runtime?.redraw();
expect(lastUniform1f).toHaveBeenCalledWith("u_intensity", 1);
expect(lastUniform1f).toHaveBeenCalledWith("u_lutIntensity", 1);
expect(lastUniform1f).toHaveBeenCalledWith("u_exposure", -2);
expect(lastUniform1f).toHaveBeenCalledWith("u_blur", 0);
expect(lastUniform1f).toHaveBeenCalledWith("u_bloom", 0.2);
expect(lastUniform1f).toHaveBeenCalledWith("u_kuwahara", 0.2);
expect(lastUniform1f).toHaveBeenCalledWith("u_pixelate", 0.3);
expect(lastUniform1f).toHaveBeenCalledWith("u_ascii", 0);
expect(lastUniform1f).toHaveBeenCalledWith("u_dither", 0.5);
});
it("initializes a zero-start grade when an animated property declares future state", () => {
const video = makeDrawableVideo();
video.setAttribute(
HF_COLOR_GRADING_ATTR,
serializeHfColorGrading({
intensity: 0,
adjust: { exposure: 0.5 },
effects: { kuwahara: 0 },
}),
);
video.style.setProperty("--hf-color-grading-intensity", "0");
video.style.setProperty("--hf-color-grading-kuwahara", "0");
document.body.appendChild(video);
runtime = createColorGradingRuntime();
expect(getContextSpy).toHaveBeenCalledTimes(1);
if (!lastUniform1f) throw new Error("Expected WebGL uniform calls");
expect(lastUniform1f).toHaveBeenCalledWith("u_intensity", 0);
expect(lastUniform1f).toHaveBeenCalledWith("u_kuwahara", 0);
expect(texImage2DCalls.some((args) => args.includes(0x8d61))).toBe(true);
video.style.setProperty("--hf-color-grading-intensity", "1");
video.style.setProperty("--hf-color-grading-kuwahara", "1");
lastUniform1f.mockClear();
runtime.redraw();
expect(lastUniform1f).toHaveBeenCalledWith("u_intensity", 1);
expect(lastUniform1f).toHaveBeenCalledWith("u_kuwahara", 1);
});
it("redraws animated still images from the transport tick", () => {
const image = makeDrawableImage();
document.body.appendChild(image);
runtime = createColorGradingRuntime();
image.style.setProperty("--hf-color-grading-blur", "0.7");
lastUniform1f?.mockClear();
expect(runtime.redrawAnimated()).toBe(1);
expect(lastUniform1f).toHaveBeenCalledWith("u_blur", 0.7);
});
it("redraws animated held video frames without duplicating active video draws", () => {
const video = makeDrawableVideo();
video.style.setProperty("--hf-color-grading-blur", "0.1");
startRuntimeWithVideo(video);
Object.defineProperty(video, "paused", { value: false, configurable: true });
Object.defineProperty(video, "ended", { value: false, configurable: true });
expect(runtime?.redrawAnimated()).toBe(0);
Object.defineProperty(video, "paused", { value: true, configurable: true });
Object.defineProperty(video, "ended", { value: true, configurable: true });
video.style.setProperty("--hf-color-grading-blur", "0.8");
lastUniform1f?.mockClear();
expect(runtime?.redrawAnimated()).toBe(1);
expect(lastUniform1f).toHaveBeenCalledWith("u_blur", 0.8);
});
it("keeps the last shader frame visible while a video seek is waiting for a drawable frame", () => {
const { video, canvas } = startRuntimeWithVideo();
@@ -373,6 +727,45 @@ describe("createColorGradingRuntime", () => {
expect(canvas.style.opacity).toBe("0.75");
});
it("allows a drawable producer render frame to initialize hidden source grading", () => {
const video = makeDrawableVideo();
video.style.display = "none";
document.body.appendChild(video);
const frame = document.createElement("img");
frame.id = "__render_frame_hero-video__";
frame.className = "__render_frame__";
frame.style.visibility = "visible";
Object.defineProperty(frame, "complete", { value: true, configurable: true });
Object.defineProperty(frame, "naturalWidth", { value: 640, configurable: true });
Object.defineProperty(frame, "naturalHeight", { value: 360, configurable: true });
video.after(frame);
runtime = createColorGradingRuntime();
expect(getContextSpy).toHaveBeenCalledTimes(1);
expect(document.querySelector("[data-hf-color-grading-canvas]")).not.toBeNull();
});
it("does not recreate an inactive clip from its hidden producer frame", () => {
const { video, canvas } = startRuntimeWithVideo();
const frame = document.createElement("img");
frame.id = "__render_frame_hero-video__";
frame.className = "__render_frame__";
frame.style.visibility = "hidden";
Object.defineProperty(frame, "complete", { value: true, configurable: true });
Object.defineProperty(frame, "naturalWidth", { value: 640, configurable: true });
Object.defineProperty(frame, "naturalHeight", { value: 360, configurable: true });
video.parentNode?.insertBefore(frame, canvas);
video.style.visibility = "hidden";
expect(runtime.setSourceVisibility(video, false)).toBe(true);
runtime.refresh();
expect(canvas.isConnected).toBe(false);
expect(getContextSpy).toHaveBeenCalledTimes(1);
expect(video.hasAttribute("data-hf-color-grading-source-hidden")).toBe(false);
});
it("moves the canvas above producer render-frame images before capture", () => {
const video = makeDrawableVideo();
Object.defineProperty(video, "readyState", {
@@ -427,20 +820,82 @@ describe("createColorGradingRuntime", () => {
it("passes finishing detail uniforms into the shader", () => {
const video = makeDrawableVideo();
const details = {
vignette: 0.4,
vignetteMidpoint: 0.35,
vignetteRoundness: -0.25,
vignetteFeather: 0.8,
grain: 0.2,
grainSize: 0.7,
grainRoughness: 0.3,
};
const effects = {
blur: 0.3,
pixelate: 0.1,
chromaBleed: 0.25,
tapeDamage: 0.35,
tapeTracking: 0.45,
tapeNoise: 0.55,
tapeSpeed: 0.65,
filmArtifacts: 0.45,
halftone: 0.55,
halftoneSize: 0.65,
twoInkPrint: 0.75,
twoInkPrintSize: 0.85,
ascii: 0.6,
asciiSize: 0.4,
asciiInvert: 1,
dither: 0.7,
ditherSize: 0.3,
asciiStyle: 4,
asciiColor: 1,
asciiRotation: 1,
monoScreen: 0.2,
monoScreenSize: 0.3,
monoScreenAngle: 0.4,
monoScreenSpread: 0.5,
monoScreenShape: 3,
monoScreenInvert: 1,
scanlines: 0.25,
scanlineCount: 0.35,
scanlineSoftness: 0.45,
chromaticAberration: 0.3,
chromaticAngle: 0.4,
crtCurvature: 0.2,
digitalGlitch: 0.35,
digitalGlitchColorSplit: 0.4,
digitalGlitchLineTear: 0.45,
digitalGlitchPixelate: 0.5,
digitalGlitchBlockAmount: 0.55,
digitalGlitchBlockDisplacement: 0.65,
digitalGlitchBlockOpacity: 0.15,
digitalGlitchSpeed: 0.75,
engraving: 0.8,
engravingSpacing: 0.41,
engravingMinThickness: 0.2,
engravingMaxThickness: 0.46,
engravingAngle: 0.25,
engravingContrast: 0.47,
engravingSharpness: 0.59,
engravingWave: 0.2,
engravingWaveFrequency: 0.22,
crosshatch: 0.85,
crosshatchSpacing: 0.28,
crosshatchThickness: 0.25,
crosshatchAngle: 0.25,
crosshatchContrast: 0.33,
crosshatchEdges: 0.5,
crosshatchLineWeight: 0.15,
crosshatchWave: 0.33,
crosshatchWaveFrequency: 0.22,
};
video.setAttribute(
HF_COLOR_GRADING_ATTR,
serializeHfColorGrading({
adjust: { vibrance: 0.35 },
details: {
vignette: 0.4,
vignetteMidpoint: 0.35,
vignetteRoundness: -0.25,
vignetteFeather: 0.8,
grain: 0.2,
grainSize: 0.7,
grainRoughness: 0.3,
},
effects: { blur: 0.3, pixelate: 0.1 },
details,
effects,
palette: ["#080717", "#3c185f", "#d9339f", "#ff6b66"],
}),
);
document.body.appendChild(video);
@@ -448,17 +903,192 @@ describe("createColorGradingRuntime", () => {
runtime = createColorGradingRuntime();
if (!lastUniform1f) throw new Error("Expected WebGL uniform calls");
expect(lastUniform1f).toHaveBeenCalledWith("u_vibrance", 0.35);
expect(lastUniform1f).toHaveBeenCalledWith("u_vignette", 0.4);
expect(lastUniform1f).toHaveBeenCalledWith("u_vignetteMidpoint", 0.35);
expect(lastUniform1f).toHaveBeenCalledWith("u_vignetteRoundness", -0.25);
expect(lastUniform1f).toHaveBeenCalledWith("u_vignetteFeather", 0.8);
expect(lastUniform1f).toHaveBeenCalledWith("u_grain", 0.2);
expect(lastUniform1f).toHaveBeenCalledWith("u_grainSize", 0.7);
expect(lastUniform1f).toHaveBeenCalledWith("u_grainRoughness", 0.3);
for (const [key, value] of Object.entries({ vibrance: 0.35, ...details, ...effects })) {
expect(lastUniform1f).toHaveBeenCalledWith(`u_${key}`, value);
}
expect(lastUniform1f).toHaveBeenCalledWith("u_grainSeed", expect.any(Number));
expect(lastUniform1f).toHaveBeenCalledWith("u_blur", 0.3);
expect(lastUniform1f).toHaveBeenCalledWith("u_pixelate", 0.1);
expect(lastUniform1f).toHaveBeenCalledWith("u_paletteSize", 4);
if (!lastUniform3f) throw new Error("Expected WebGL palette uniform calls");
expect(lastUniform3f).toHaveBeenCalledWith("u_palette0", 8 / 255, 7 / 255, 23 / 255);
expect(lastUniform3f).toHaveBeenCalledWith("u_palette3", 1, 107 / 255, 102 / 255);
const fragment = lastShaderSources.find((source) => source.includes("sampleMedia"));
expect(fragment).toContain("float centerLuma = lumaOf(base.rgb);");
expect(fragment).toContain("vec3(centerLuma) + blurredChroma");
expect(fragment).toContain("float headSwitch");
expect(fragment).toContain("float tapeTrackingBand");
expect(fragment).toContain("float tapeTime = u_effectTime");
expect(fragment).toContain("float tapeFrame = floor(tapeTime * 60.0);");
expect(fragment).not.toContain(
"tapeTrackingBand(float y, float center, float width, float phase)",
);
expect(fragment).toContain("float lineJitter = (digitalHash");
expect(fragment).toContain("float dustMask");
expect(fragment).toContain("float screenDot");
expect(fragment).toContain("vec3 applyHalftone");
expect(fragment).toContain("vec3 applyTwoInkPrint");
expect(fragment).toContain("float standardAsciiSample");
expect(fragment).toContain("float bayer4");
expect(fragment).toContain("vec3 applyAscii");
expect(fragment).toContain("vec3 applyDither");
expect(fragment).toContain("vec3 applyMonoScreen");
expect(fragment).toContain("vec2 applyCrtWarp");
expect(fragment).toContain("vec4 sampleChromaticMedia");
expect(fragment).toContain("vec3 applyScanlines");
expect(fragment).toContain("vec3 applyDigitalGlitch");
expect(fragment).toContain("vec3 applyEngraving");
expect(fragment).toContain("vec3 applyCrosshatch");
expect(fragment).toContain("float crosshatchEdge");
expect(fragment).toContain("vec3 applyCrosshatch(vec2 uv");
expect(fragment).toContain("crosshatchEdge(uv)");
expect(fragment).not.toContain("crosshatchEdge(v_uv)");
expect(fragment).toContain("vec2 texel = 1.0 / max(u_resolution * u_uvScale, vec2(1.0));");
expect(fragment).toContain("float baseAngle = -clamp(u_crosshatchAngle, 0.0, 1.0) * PI;");
expect(fragment).toContain("float variation = mix(1.0, 0.5 + digitalHash");
expect(fragment).toContain("baseAngle + PI * 0.5");
expect(fragment).toContain("baseAngle + PI * 0.25");
expect(fragment).toContain("baseAngle - PI * 0.25");
expect(fragment).toContain("asciiStyleSample(floor(u_asciiStyle + 0.5)");
expect(fragment).toContain("float cellHeight = mix(4.0, 80.0");
expect(fragment).toContain("vec2 asciiEdgeDirection");
expect(fragment).toContain("vec3 background = paletteColor(0.0);");
expect(fragment).not.toContain("grainHash(cell + vec2(17.0, 43.0))");
expect(fragment).toContain("sampleColor = sampleChromaticMedia(uv, sampleColor);");
expect(fragment).toContain("sampleColor.rgb = applyDigitalGlitch(uv, sampleColor.rgb);");
expect(fragment).toContain("vec3 sampleDigitalSplit");
expect(fragment).toContain("float digitalHash(vec2 p)");
expect(fragment).toContain("float direction = digitalHash");
expect(fragment).toContain("float randomA = digitalHash");
expect(fragment).toContain("float colorSplit = clamp(u_digitalGlitchColorSplit");
expect(fragment).toContain("float pixelate = clamp(u_digitalGlitchPixelate");
expect(fragment).toContain("blockDisplacement > 0.0 && blockOpacity > 0.0");
expect(fragment).toContain("displaced = mix(uv, displaced, blockOpacity);");
expect(fragment).not.toContain("vec3 electronicBlock");
expect(fragment).toContain("color = applyMonoScreen");
expect(fragment).toContain("color = applyEngraving");
expect(fragment).toContain("color = applyCrosshatch");
expect(fragment).toContain("color = applyScanlines");
expect(fragment).toContain("float curvature = clamp(u_crtCurvature, 0.0, 1.0) * 0.5;");
expect(fragment).toContain("float dist = dot(centered, centered);");
expect(fragment).toContain("centered *= 1.0 + curvature * dist;");
expect(fragment).toContain("float wave = 0.5 + 0.5 * sin(v_uv.y * count * PI);");
expect(fragment).toContain("float line = mix(1.0 - wave, pow(1.0 - wave, 2.2), softness);");
expect(fragment).not.toContain("float hardLine = step(0.78, wave);");
expect(fragment.indexOf("float vignettePower")).toBeGreaterThan(
fragment.indexOf("color = applyScanlines"),
);
expect(fragment).toContain("amount * 0.02");
expect(fragment).not.toContain("mix(center.rgb, split, amount)");
});
it("renders Kuwahara through bounded moment passes before the main shader", () => {
const video = makeDrawableVideo();
video.setAttribute(
HF_COLOR_GRADING_ATTR,
serializeHfColorGrading({
effects: {
kuwahara: 0.8,
kuwaharaRadius: 0.25,
kuwaharaSharpness: 0.4,
kuwaharaSaturation: 0.6,
},
}),
);
document.body.appendChild(video);
runtime = createColorGradingRuntime();
if (!lastUniform1f) throw new Error("Expected WebGL uniform calls");
expect(lastUniform1f).toHaveBeenCalledWith("u_kuwahara", 0.8);
expect(lastUniform1f).toHaveBeenCalledWith("u_kuwaharaRadius", 0.25);
expect(lastUniform1f).toHaveBeenCalledWith("u_kuwaharaSharpness", 0.4);
expect(lastUniform1f).toHaveBeenCalledWith("u_kuwaharaSaturation", 0.6);
expect(lastShaderSources.some((source) => source.includes("u_kuwaharaMoments"))).toBe(true);
expect(
lastShaderSources.some((source) => source.includes("meanSquare - dot(mean, mean)")),
).toBe(true);
expect(texImage2DCalls.some((args) => args.includes(0x8d61))).toBe(true);
});
it("falls back to the untreated shader output when half-float targets are unavailable", () => {
getContextSpy.mockImplementation((type: string) =>
type === "webgl" ? createMockWebGl({ halfFloatSupported: false }) : null,
);
const video = makeDrawableVideo();
video.setAttribute(
HF_COLOR_GRADING_ATTR,
serializeHfColorGrading({ effects: { kuwahara: 1 } }),
);
const { canvas } = startRuntimeWithVideo(video);
if (!lastUniform1f) throw new Error("Expected WebGL uniform calls");
expect(lastUniform1f).toHaveBeenCalledWith("u_kuwaharaReady", 0);
expect(canvas.style.display).toBe("block");
expect(runtime?.getStatus(video)).toEqual({
state: "unavailable",
message: "Kuwahara requires half-float framebuffer support",
});
expect(texImage2DCalls.some((args) => args.includes(0x8d61))).toBe(false);
});
it("releases lazy Kuwahara GPU resources when attributed media becomes inactive", () => {
const video = makeDrawableVideo();
video.setAttribute(
HF_COLOR_GRADING_ATTR,
serializeHfColorGrading({ effects: { kuwahara: 1 } }),
);
startRuntimeWithVideo(video);
const gl = getContextSpy.mock.results[0]?.value as WebGLRenderingContext;
expect(runtime?.setSourceVisibility(video, false)).toBe(true);
expect(gl.deleteFramebuffer).toHaveBeenCalledTimes(2);
expect(gl.deleteTexture).toHaveBeenCalledTimes(2);
expect(gl.deleteProgram).toHaveBeenCalledTimes(2);
expect(loseContextCalls).toBe(0);
expect(document.querySelector("[data-hf-color-grading-canvas]")).toBeNull();
});
it("releases pooled WebGL contexts when the runtime is destroyed", () => {
const { video } = startRuntimeWithVideo();
expect(runtime?.setSourceVisibility(video, false)).toBe(true);
expect(loseContextCalls).toBe(0);
runtime?.destroy();
runtime = null;
expect(loseContextCalls).toBe(1);
});
it("drives temporal shader effects from canonical composition time", () => {
let playerTime = 2.25;
Object.defineProperty(window, "__player", {
value: { getTime: () => playerTime },
configurable: true,
});
const video = makeDrawableVideo();
Object.defineProperty(video, "currentTime", { value: 19.5, configurable: true });
video.setAttribute(
HF_COLOR_GRADING_ATTR,
serializeHfColorGrading({ effects: { digitalGlitch: 1 } }),
);
document.body.appendChild(video);
runtime = createColorGradingRuntime();
if (!lastUniform1f) throw new Error("Expected WebGL uniform calls");
expect(lastUniform1f).toHaveBeenCalledWith("u_effectTime", 2.25);
const firstGrainSeed = lastUniform1f.mock.calls.findLast(
([uniform]) => uniform === "u_grainSeed",
)?.[1] as number | undefined;
playerTime = 2.5;
runtime.redraw();
const secondGrainSeed = lastUniform1f.mock.calls.findLast(
([uniform]) => uniform === "u_grainSeed",
)?.[1] as number | undefined;
expect(firstGrainSeed).toBeTypeOf("number");
expect(secondGrainSeed).toBe((firstGrainSeed ?? 0) + 15);
const fragment = lastShaderSources.find((source) => source.includes("sampleMedia"));
expect(fragment).toContain("float time = u_effectTime * speed;");
});
it("uses the effected media sample as the graded shader input", () => {
@@ -480,7 +1110,11 @@ describe("createColorGradingRuntime", () => {
expect(fragment).toContain("float grainPixelSize");
expect(fragment).toContain("float blackPoint = clamp(u_blacks * 0.18");
expect(fragment).toContain("float whitePoint = clamp(1.0 - u_whites * 0.18");
expect(fragment).toContain("vec3 color = sampleColor.rgb * pow(2.0, u_exposure);");
expect(fragment).toContain("vec3 applyPrimaryGrade(vec3 color)");
expect(fragment).toContain(
"vec3 color = mix(sampleColor.rgb, applyPrimaryGrade(sampleColor.rgb), u_intensity);",
);
expect(fragment).not.toContain("mix(original, color, u_intensity)");
expect(fragment).not.toContain("sampleSoft");
const blurFragment = lastShaderSources.find((source) =>
source.includes("uniform vec2 u_direction;"),
@@ -505,6 +1139,47 @@ describe("createColorGradingRuntime", () => {
).toBe(true);
});
it("renders article bloom with thresholded half-resolution Gaussian passes", () => {
const video = makeDrawableVideo();
video.setAttribute(
HF_COLOR_GRADING_ATTR,
serializeHfColorGrading({ effects: { bloom: 0.5, bloomRadius: 8 } }),
);
document.body.appendChild(video);
runtime = createColorGradingRuntime();
expect(lastUniform1f).toHaveBeenCalledWith("u_bloom", 0.5);
expect(lastUniform1f).toHaveBeenCalledWith("u_bloomReady", 1);
expect(lastUniform1f).toHaveBeenCalledWith("u_radius", 4);
expect(texImage2DCalls.some((args) => args[3] === 320 && args[4] === 180)).toBe(true);
const bloomFragment = lastShaderSources.find((source) => source.includes("u_bloomPass"));
expect(bloomFragment).toContain("vec3(0.299, 0.587, 0.114)");
expect(bloomFragment).toContain("0.227027");
expect(bloomFragment).toContain("0.1945946");
expect(bloomFragment).toContain("0.016216");
});
it("releases the lazy bloom output used only when blur and bloom coexist", () => {
const video = makeDrawableVideo();
video.setAttribute(
HF_COLOR_GRADING_ATTR,
serializeHfColorGrading({
effects: { blur: 0.4, bloom: 0.5, bloomRadius: 8 },
}),
);
startRuntimeWithVideo(video);
const gl = getContextSpy.mock.results[0]?.value as WebGLRenderingContext;
expect(texImage2DCalls.some((args) => args[3] === 640 && args[4] === 360)).toBe(true);
expect(texImage2DCalls.some((args) => args[3] === 320 && args[4] === 180)).toBe(true);
expect(runtime?.setSourceVisibility(video, false)).toBe(true);
expect(gl.deleteFramebuffer).toHaveBeenCalledTimes(3);
expect(gl.deleteTexture).toHaveBeenCalledTimes(3);
expect(gl.deleteProgram).toHaveBeenCalledTimes(1);
});
it("loads cube LUTs and enables LUT uniforms", async () => {
const fetchMock = stubCubeLutFetch();
const origin = window.location.origin;
@@ -532,6 +1207,40 @@ describe("createColorGradingRuntime", () => {
expect(runtime.getStatus("#hero-video").message).toBe("Shader + LUT active");
});
it("waits for active LUTs before deterministic capture", async () => {
let releaseFetch: (() => void) | undefined;
vi.stubGlobal(
"fetch",
vi.fn(
() =>
new Promise<{ ok: boolean; status: number; text: () => Promise<string> }>((resolve) => {
releaseFetch = () =>
resolve({ ok: true, status: 200, text: () => Promise.resolve(IDENTITY_2) });
}),
),
);
const video = makeDrawableVideo();
video.setAttribute(
HF_COLOR_GRADING_ATTR,
serializeHfColorGrading({ lut: { src: "assets/luts/identity.cube", intensity: 1 } }),
);
document.body.appendChild(video);
runtime = createColorGradingRuntime();
expect(runtime.getStatus(video)).toEqual({ state: "pending", message: "Loading LUT" });
const ready = runtime.waitForActiveLuts();
let settled = false;
void ready.then(() => {
settled = true;
});
await Promise.resolve();
expect(settled).toBe(false);
releaseFetch?.();
expect(await ready).toBe(1);
expect(runtime.getStatus(video).message).toBe("Shader + LUT active");
});
it("bounds the runtime LUT cache", async () => {
const fetchMock = stubCubeLutFetch();
const origin = window.location.origin;
@@ -644,15 +1353,16 @@ describe("installAuthoredOpacityCapture", () => {
el.remove();
});
it("stamps an already-inserted element the moment it GAINS grading at runtime", async () => {
it("stamps ungraded media before a live Studio grade can hide it", async () => {
installAuthoredOpacityCapture();
const el = document.createElement("img");
el.style.opacity = "0.9";
document.body.appendChild(el);
await Promise.resolve();
expect(el.hasAttribute("data-hf-authored-opacity")).toBe(false);
expect(el.getAttribute("data-hf-authored-opacity")).toBe("0.9");
// Studio applies a preset to a previously ungraded element — no re-insert.
// The live runtime hides the source before Studio persists the attribute.
el.style.setProperty("opacity", "0", "important");
el.setAttribute(HF_COLOR_GRADING_ATTR, serializeHfColorGrading({ adjust: { exposure: 0.5 } }));
await Promise.resolve();
expect(el.getAttribute("data-hf-authored-opacity")).toBe("0.9");
File diff suppressed because it is too large Load Diff
+77
View File
@@ -1320,6 +1320,53 @@ describe("initSandboxRuntimeModular", () => {
expect(video.style.visibility).toBe("hidden");
});
it("allocates color grading only for the active timed media", () => {
const getContextSpy = vi.spyOn(HTMLCanvasElement.prototype, "getContext").mockReturnValue(null);
const root = document.createElement("div");
root.setAttribute("data-composition-id", "main");
root.setAttribute("data-root", "true");
root.setAttribute("data-start", "0");
root.setAttribute("data-duration", "4");
root.setAttribute("data-width", "1920");
root.setAttribute("data-height", "1080");
document.body.appendChild(root);
const futureComposition = document.createElement("div");
futureComposition.id = "future-composition";
futureComposition.setAttribute("data-start", "2");
root.appendChild(futureComposition);
for (const [id, start] of [
["first", "0"],
["second", "2"],
]) {
const video = document.createElement("video");
video.id = id;
video.setAttribute("data-start", start);
video.setAttribute("data-duration", "2");
video.setAttribute("data-color-grading", '{"adjust":{"exposure":0.1}}');
Object.defineProperty(video, "paused", { value: true, configurable: true });
Object.defineProperty(video, "readyState", { value: 0, configurable: true });
video.load = () => {};
root.appendChild(video);
}
window.__timelines = { main: createMockTimeline(4) };
initSandboxRuntimeModular();
expect(getContextSpy).toHaveBeenCalledTimes(1);
expect(document.getElementById("first")?.style.visibility).toBe("visible");
expect(document.getElementById("second")?.style.visibility).toBe("hidden");
expect(futureComposition.style.visibility).toBe("");
expect(futureComposition.style.display).toBe("");
window.__player?.seek(3);
expect(getContextSpy).toHaveBeenCalledTimes(2);
expect(document.getElementById("first")?.style.visibility).toBe("hidden");
expect(document.getElementById("second")?.style.visibility).toBe("visible");
});
it("plays scheduled child timelines without a captured root timeline when audio has failed", () => {
const raf = createManualRaf();
vi.spyOn(performance, "now").mockImplementation(() => raf.now());
@@ -2122,6 +2169,36 @@ describe("initSandboxRuntimeModular", () => {
expect(seekTimes.length).toBeGreaterThan(beforeResume);
});
it("redraws animated grading from the transport clock only during playback", () => {
const raf = createManualRaf();
vi.spyOn(performance, "now").mockImplementation(() => raf.now());
window.requestAnimationFrame = raf.requestAnimationFrame as typeof window.requestAnimationFrame;
window.cancelAnimationFrame = raf.cancelAnimationFrame as typeof window.cancelAnimationFrame;
document.body.innerHTML = `
<div data-composition-id="root" data-duration="5" data-width="1920" data-height="1080"></div>
`;
window.__timelines = { root: createMockTimeline(5) };
initSandboxRuntimeModular();
const runtime = (
window as Window & { __hf?: { colorGrading?: { redrawAnimated: () => number } } }
).__hf?.colorGrading;
if (!runtime) throw new Error("Expected color grading runtime");
const redrawAnimated = vi.spyOn(runtime, "redrawAnimated");
raf.step(16);
expect(redrawAnimated).not.toHaveBeenCalled();
window.__player?.play();
raf.step(16);
expect(redrawAnimated).toHaveBeenCalledTimes(1);
window.__player?.pause();
raf.step(16);
expect(redrawAnimated).toHaveBeenCalledTimes(1);
});
it("keeps a usable bound timeline when the registry entry is replaced", () => {
const raf = createManualRaf();
vi.spyOn(performance, "now").mockImplementation(() => raf.now());
+11 -2
View File
@@ -1793,8 +1793,10 @@ export function initSandboxRuntimeModular(): void {
const dataHiddenDisplayRestores = new WeakMap<HTMLElement, string>();
const dataHiddenDisplayNodes = new WeakSet<HTMLElement>();
const syncTimedElementVisibility = (currentTime: number) => {
const visibilityNodes = Array.from(document.querySelectorAll("[data-start]"));
const syncTimedElementVisibility = (
currentTime: number,
visibilityNodes: Element[] = Array.from(document.querySelectorAll("[data-start]")),
) => {
const rootComp = resolveRootCompositionElement();
for (const rawNode of visibilityNodes) {
if (!(rawNode instanceof HTMLElement)) continue;
@@ -2174,6 +2176,10 @@ export function initSandboxRuntimeModular(): void {
});
picker.installPickerApi();
syncTimedElementVisibility(
state.currentTime,
Array.from(document.querySelectorAll("video[data-start], img[data-start]")),
);
const colorGrading = createColorGradingRuntime();
colorGradingRuntime = colorGrading;
registerRuntimeCleanup(() => {
@@ -2839,6 +2845,9 @@ export function initSandboxRuntimeModular(): void {
if (clock.isPlaying() || !hasActiveStudioManualEditGesture()) {
seekTimelineAndAdapters(t);
}
if (clock.isPlaying()) {
colorGrading.redrawAnimated();
}
// Looping is handled at the player layer (<hyperframes-player>),
// not the runtime. The clock pauses at duration; GSAP's repeat:-1
@@ -2356,6 +2356,14 @@ async function prepareFrameForCapture(
if (session.onBeforeCapture) {
await session.onBeforeCapture(page, quantizedTime);
}
await page.evaluate(async () => {
const runtime = (
window as Window & {
__hf?: { colorGrading?: { waitForActiveLuts?: () => Promise<number> } };
}
).__hf?.colorGrading;
await runtime?.waitForActiveLuts?.();
});
const beforeCaptureMs = Date.now() - beforeCaptureStart;
// Page-side compositing three-phase protocol:
@@ -181,9 +181,12 @@ describe("injectVideoFramesBatch replacement layout", () => {
'<html><body><div id="root"><video id="clip" style="position:absolute;inset:0;width:100%;height:100%;object-fit:cover"></video></div></body></html>',
);
const events: string[] = [];
Object.defineProperty(window.HTMLImageElement.prototype, "decode", {
configurable: true,
value: () => Promise.resolve(),
value: async () => {
events.push("decode");
},
});
const video = document.getElementById("clip") as HTMLVideoElement;
@@ -232,6 +235,10 @@ describe("injectVideoFramesBatch replacement layout", () => {
const previousDocument = globals.document;
globals.window = window;
globals.document = document;
const redraw = vi.fn(() => events.push("redraw"));
(window as unknown as { __hf: { colorGrading: { redraw: () => void } } }).__hf = {
colorGrading: { redraw },
};
try {
const page = {
evaluate: async (
@@ -266,6 +273,8 @@ describe("injectVideoFramesBatch replacement layout", () => {
expect(img?.style.right).toBe("auto");
expect(img?.style.bottom).toBe("auto");
expect(img?.style.inset).toBe("auto");
expect(redraw).toHaveBeenCalledOnce();
expect(events).toEqual(["decode", "redraw"]);
});
});
@@ -569,6 +578,10 @@ describe("video-frame injection respects ancestor visibility", () => {
seededImg.classList.add("__render_frame__");
seededImg.style.opacity = "0";
setup.video.parentNode?.insertBefore(seededImg, setup.video.nextSibling);
const setSourceVisibility = vi.fn();
(setup.window as unknown as { __hf: unknown }).__hf = {
colorGrading: { setSourceVisibility },
};
try {
await syncVideoFrameVisibility(passthroughPage(), ["pip"]);
@@ -578,6 +591,7 @@ describe("video-frame injection respects ancestor visibility", () => {
expect(seededImg.style.opacity).toBe("1");
expect(seededImg.style.visibility).toBe("visible");
expect(setSourceVisibility).toHaveBeenCalledWith(setup.video, true);
});
it("syncVideoFrameVisibility shows the replacement <img> when a plain [data-start] host is visibility:hidden", async () => {
@@ -643,6 +657,10 @@ describe("video-frame injection respects ancestor visibility", () => {
seededImg.style.visibility = "visible";
setup.video.parentNode?.insertBefore(seededImg, setup.video.nextSibling);
const setPropertySpy = vi.spyOn(seededImg.style, "setProperty");
const setSourceVisibility = vi.fn();
(setup.window as unknown as { __hf: unknown }).__hf = {
colorGrading: { setSourceVisibility },
};
try {
await syncVideoFrameVisibility(passthroughPage(), ["pip"]);
@@ -652,6 +670,7 @@ describe("video-frame injection respects ancestor visibility", () => {
expect(seededImg.style.visibility).toBe("hidden");
expect(setPropertySpy).toHaveBeenCalledWith("visibility", "hidden", "important");
expect(setSourceVisibility).toHaveBeenCalledWith(setup.video, false);
});
it("applyDomLayerMask does not revive hidden idless timed descendants of a shown layer", async () => {
@@ -8,13 +8,13 @@
// fallow-ignore-file code-duplication
import { type Page } from "puppeteer-core";
import { type CaptureOptions } from "../types.js";
import { COLOR_GRADING_SOURCE_HIDDEN_ATTR } from "@hyperframes/core/color-grading";
import {
HF_COLOR_GRADING_CANVAS_ID_PREFIX,
MEDIA_VISUAL_STYLE_PROPERTIES,
} from "@hyperframes/core";
export const cdpSessionCache = new WeakMap<Page, import("puppeteer-core").CDPSession>();
const COLOR_GRADING_SOURCE_HIDDEN_ATTR = "data-hf-color-grading-source-hidden";
export async function getCdpSession(page: Page): Promise<import("puppeteer-core").CDPSession> {
let client = cdpSessionCache.get(page);
@@ -789,6 +789,11 @@ export async function injectVideoFramesBatch(
if (pendingDecodes.length > 0) {
await Promise.all(pendingDecodes);
}
if (injectedIds.length > 0) {
const redraw = (window as Window & { __hf?: { colorGrading?: { redraw?: () => void } } })
.__hf?.colorGrading?.redraw;
redraw?.();
}
return injectedIds;
},
updates,
@@ -825,6 +830,13 @@ export async function syncVideoFrameVisibility(
return false;
};
const active = new Set(ids);
const setColorGradingVisibility = (
window as Window & {
__hf?: {
colorGrading?: { setSourceVisibility?: (target: Element, visible: boolean) => boolean };
};
}
).__hf?.colorGrading?.setSourceVisibility;
const videos = Array.from(
document.querySelectorAll("video[data-start]"),
) as HTMLVideoElement[];
@@ -832,7 +844,8 @@ export async function syncVideoFrameVisibility(
const img = video.nextElementSibling as HTMLElement | null;
const hasImg = img && img.classList.contains("__render_frame__");
const ancestorHidden = isVisualAncestorHidden(video);
if (active.has(video.id) && !ancestorHidden) {
const visible = active.has(video.id) && !ancestorHidden;
if (visible) {
// Active video: show injected <img>, hide native <video>.
// Do NOT clobber inline opacity here — GSAP-controlled opacity must
// survive until injectVideoFramesBatch reads it via getComputedStyle.
@@ -859,6 +872,7 @@ export async function syncVideoFrameVisibility(
img.style.setProperty("visibility", "hidden", "important");
}
}
setColorGradingVisibility?.(video, visible);
}
},
activeVideoIds,
@@ -148,25 +148,6 @@ function createFrameSourceCache(
export const __testing = { createFrameSourceCache };
async function redrawRuntimeColorGrading(page: Page): Promise<void> {
await page.evaluate(() => {
const hf = (
window as Window & {
__hf?: {
colorGrading?: { redraw?: () => unknown };
};
}
).__hf;
const redraw = hf?.colorGrading?.redraw;
if (typeof redraw !== "function") return;
try {
redraw();
} catch {
// Optional page-side shader layer.
}
});
}
/**
* Creates a BeforeCaptureHook that injects pre-extracted video frames
* into the page, replacing native <video> elements with frame images.
@@ -248,7 +229,6 @@ export function createVideoFrameInjector(
}, time);
}
}
await redrawRuntimeColorGrading(page);
};
}
+108
View File
@@ -81,6 +81,114 @@ describe("media rules", () => {
expect(finding).toBeUndefined();
});
it("reports grading controls placed outside their schema sections", async () => {
const html = `
<html><body>
<div id="root" data-composition-id="c1" data-width="1920" data-height="1080">
<video id="v1" data-start="0" data-duration="5" src="clip.mp4" muted data-color-grading='{"preset":"skin-soft","intensity":0.58,"highlights":-0.06,"temperature":0.02}'></video>
</div>
<script>window.__timelines = {};</script>
</body></html>`;
const result = await lintHyperframeHtml(html);
const finding = result.findings.find((f) => f.code === "color_grading_invalid_structure");
expect(finding?.severity).toBe("error");
expect(finding?.message).toContain("highlights");
expect(finding?.fixHint).toContain('"adjust"');
});
it("accepts grading controls inside their schema sections", async () => {
const html = `
<html><body>
<div id="root" data-composition-id="c1" data-width="1920" data-height="1080">
<video id="v1" data-start="0" data-duration="5" src="clip.mp4" muted data-color-grading='{"preset":"skin-soft","intensity":0.58,"adjust":{"highlights":-0.06,"temperature":0.02},"details":{"vignette":0.03},"effects":{"blur":0.1,"chromaBleed":0.2,"tapeDamage":0.3,"tapeTracking":0.4,"tapeNoise":0.5,"tapeSpeed":0.6,"filmArtifacts":0.4,"halftone":0.5,"halftoneSize":0.6,"twoInkPrint":0.7,"twoInkPrintSize":0.8,"ascii":0.9,"asciiSize":0.4,"asciiInvert":1,"dither":0.8,"ditherSize":0.3,"bloom":0.5,"bloomRadius":8,"asciiStyle":4,"asciiColor":1,"asciiRotation":1,"monoScreen":0.5,"monoScreenSize":0.4,"monoScreenAngle":0.3,"monoScreenSpread":0.2,"monoScreenShape":3,"monoScreenInvert":1,"scanlines":0.3,"scanlineCount":0.4,"scanlineSoftness":0.5,"chromaticAberration":0.2,"chromaticAngle":0.6,"crtCurvature":0.25,"digitalGlitch":0.4,"digitalGlitchColorSplit":0.45,"digitalGlitchLineTear":0.5,"digitalGlitchPixelate":0.55,"digitalGlitchBlockAmount":0.6,"digitalGlitchBlockDisplacement":0.7,"digitalGlitchBlockOpacity":0.2,"digitalGlitchSpeed":0.7,"engraving":1,"engravingSpacing":0.4117647,"engravingMinThickness":0.2,"engravingMaxThickness":0.4571429,"engravingAngle":0.25,"engravingContrast":0.4666667,"engravingSharpness":0.59,"engravingWave":0.2,"engravingWaveFrequency":0.2222222},"palette":["#ff6b66","#080717","#d9339f","#3c185f"],"lut":null}'></video>
</div>
<script>window.__timelines = {};</script>
</body></html>`;
const result = await lintHyperframeHtml(html);
expect(result.findings.find((f) => f.code.startsWith("color_grading_"))).toBeUndefined();
});
it("accepts crosshatch controls in the effects section", async () => {
const html = `
<html><body>
<div id="root" data-composition-id="c1" data-width="1920" data-height="1080">
<video data-start="0" data-duration="5" src="clip.mp4" muted data-color-grading='{"effects":{"crosshatch":1,"crosshatchSpacing":0.28,"crosshatchThickness":0.25,"crosshatchAngle":0.25,"crosshatchContrast":0.3333333,"crosshatchEdges":0.5,"crosshatchLineWeight":0,"crosshatchWave":0.33,"crosshatchWaveFrequency":0.2222222}}'></video>
</div>
<script>window.__timelines = {};</script>
</body></html>`;
const result = await lintHyperframeHtml(html);
expect(result.findings.find((f) => f.code.startsWith("color_grading_"))).toBeUndefined();
});
it("accepts Kuwahara controls in the effects section", async () => {
const html = `
<html><body>
<div id="root" data-composition-id="c1" data-width="1920" data-height="1080">
<video data-start="0" data-duration="5" src="clip.mp4" muted data-color-grading='{"effects":{"kuwahara":1,"kuwaharaRadius":0.142857,"kuwaharaSharpness":0.3125,"kuwaharaSaturation":0.5}}'></video>
</div>
<script>window.__timelines = {};</script>
</body></html>`;
const result = await lintHyperframeHtml(html);
expect(result.findings.find((f) => f.code.startsWith("color_grading_"))).toBeUndefined();
});
it("reports malformed or out-of-range color grading palettes", async () => {
const html = `
<html><body>
<div id="root" data-composition-id="c1" data-width="1920" data-height="1080">
<video id="v1" data-start="0" data-duration="5" src="clip.mp4" muted data-color-grading='{"effects":{"dither":1},"palette":["#000000","red"]}'></video>
</div>
<script>window.__timelines = {};</script>
</body></html>`;
const result = await lintHyperframeHtml(html);
const finding = result.findings.find((f) => f.code === "color_grading_invalid_structure");
expect(finding?.severity).toBe("error");
expect(finding?.fixHint).toContain("2 to 6");
expect(finding?.fixHint).toContain("#RRGGBB");
});
it("reports malformed grading JSON", async () => {
const html = `
<html><body>
<div id="root" data-composition-id="c1" data-width="1920" data-height="1080">
<video id="v1" data-start="0" data-duration="5" src="clip.mp4" muted data-color-grading='{"preset":"skin-soft"'></video>
</div>
<script>window.__timelines = {};</script>
</body></html>`;
const result = await lintHyperframeHtml(html);
expect(result.findings.find((f) => f.code === "color_grading_invalid_json")?.severity).toBe(
"error",
);
});
it("reports invalid string values for structured grading sections", async () => {
const html = `
<html><body>
<div id="root" data-composition-id="c1" data-width="1920" data-height="1080">
<video id="v1" data-start="0" data-duration="5" src="clip.mp4" muted data-color-grading='{"adjust":"cinematic"}'></video>
</div>
<script>window.__timelines = {};</script>
</body></html>`;
const result = await lintHyperframeHtml(html);
expect(
result.findings.find((f) => f.code === "color_grading_invalid_structure")?.severity,
).toBe("error");
});
it("reports color grading on non-media elements", async () => {
const html = `
<html><body>
<div id="root" data-composition-id="c1" data-width="1920" data-height="1080">
<div id="background" data-color-grading='{"preset":"skin-soft"}'></div>
</div>
<script>window.__timelines = {};</script>
</body></html>`;
const result = await lintHyperframeHtml(html);
expect(result.findings.find((f) => f.code === "color_grading_non_media")?.severity).toBe(
"error",
);
});
it("reports warning for media with preload=none", async () => {
const html = `
<html><body>
+54
View File
@@ -1,5 +1,6 @@
import type { LintContext, HyperframeLintFinding } from "../context";
import { readAttr, readDecodedAttr, truncateSnippet, isMediaTag } from "../utils";
import { validateColorGradingContract } from "@hyperframes/parsers/color-grading-contract";
function escapeRegExp(value: string): string {
return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
@@ -247,6 +248,59 @@ export const mediaRules: Array<(ctx: LintContext) => HyperframeLintFinding[]> =
return findings;
},
// color_grading_* — grading is a structured media-only contract. Unknown
// keys are ignored by the runtime, so catch them before an agent can report
// controls that never actually rendered.
({ tags }) => {
const findings: HyperframeLintFinding[] = [];
for (const tag of tags) {
const raw = readDecodedAttr(tag.raw, "data-color-grading");
if (raw === null) continue;
const elementId = readAttr(tag.raw, "id") || undefined;
const report = (code: string, message: string, fixHint: string) => {
findings.push({
code,
severity: "error",
message,
elementId,
fixHint,
snippet: truncateSnippet(tag.raw),
});
};
if (tag.name !== "video" && tag.name !== "img") {
report(
"color_grading_non_media",
`data-color-grading on <${tag.name}> has no effect. The shader runtime only grades real <video> and <img> elements.`,
"Move the grading attribute to the real <video> or <img> media element. Do not attach it to a wrapper or CSS background.",
);
continue;
}
const trimmed = raw.trim();
if (!trimmed.startsWith("{") && !trimmed.startsWith("[")) continue;
let parsed: unknown;
try {
parsed = JSON.parse(trimmed);
} catch {
report(
"color_grading_invalid_json",
"data-color-grading contains malformed JSON and will not render.",
'Use valid JSON, for example {"preset":"skin-soft","intensity":0.6,"adjust":{"highlights":-0.08}}.',
);
continue;
}
for (const issue of validateColorGradingContract(parsed)) {
report(
"color_grading_invalid_structure",
`data-color-grading ${issue.path} ${issue.message}.`,
issue.hint ??
"Use the documented media-treatment contract and correct or remove the invalid value.",
);
}
}
return findings;
},
// video_missing_muted
({ tags }) => {
const findings: HyperframeLintFinding[] = [];
+27 -3
View File
@@ -139,6 +139,20 @@ export function isColorGradingVariableRef(value: unknown): value is string {
return typeof value === "string" && VARIABLE_REF.test(value.trim());
}
function unknownKeysHint(path: string, unknown: readonly string[]): string {
if (path !== "grading") return `Correct or remove the unsupported "${path}" keys.`;
const sections = new Set(
unknown.flatMap((key) =>
OBJECT_SECTIONS.filter(([, keys]) => (keys as readonly string[]).includes(key)).map(
([section]) => section,
),
),
);
return sections.size === 1
? `Move those controls under "${[...sections][0]}".`
: "Use only the documented media-treatment keys at the top level.";
}
function validateObject(
value: unknown,
path: string,
@@ -153,7 +167,11 @@ function validateObject(
const allowed = new Set(keys);
const unknown = Object.keys(value).filter((key) => !allowed.has(key));
if (unknown.length > 0) {
issues.push({ path, message: `has unsupported key(s): ${unknown.join(", ")}` });
issues.push({
path,
message: `has unsupported key(s): ${unknown.join(", ")}`,
hint: unknownKeysHint(path, unknown),
});
}
return value;
}
@@ -192,13 +210,19 @@ function validateNumericSection(
function validatePalette(value: unknown, issues: ColorGradingContractIssue[]): void {
if (value === undefined || value === null || isColorGradingVariableRef(value)) return;
const hint =
'Use 2 to 6 colors in the intended mapping order, each written as exact "#RRGGBB", or use a project variable reference.';
if (!Array.isArray(value) || value.length < 2 || value.length > 6) {
issues.push({ path: "palette", message: "must contain 2 to 6 hex colors" });
issues.push({ path: "palette", message: "must contain 2 to 6 hex colors", hint });
return;
}
value.forEach((color, index) => {
if (typeof color !== "string" || !PALETTE_COLOR.test(color)) {
issues.push({ path: `palette[${index}]`, message: "must be a six-digit hex color" });
issues.push({
path: `palette[${index}]`,
message: "must be a six-digit hex color",
hint,
});
}
});
}
+2 -1
View File
@@ -170,13 +170,14 @@ describe("parseGsapScript", () => {
it("extracts all GSAP properties including non-standard ones", () => {
const script = `
const tl = gsap.timeline({ paused: true });
tl.to("#el1", { opacity: 1, backgroundColor: "red", x: 50, duration: 0.5 }, 0);
tl.to("#el1", { opacity: 1, backgroundColor: "red", x: 50, "--hf-color-grading-intensity": 0.5, duration: 0.5 }, 0);
`;
const result = parseGsapScript(script);
expect(result.animations[0].properties.opacity).toBe(1);
expect(result.animations[0].properties.x).toBe(50);
expect(result.animations[0].properties.backgroundColor).toBe("red");
expect(result.animations[0].properties["--hf-color-grading-intensity"]).toBe(0.5);
});
it("extracts ease from properties", () => {