ext/hyperframes
2
0
Fork 0
mirror of https://github.com/heygen-com/hyperframes.git synced 2026-09-01 19:42:03 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(runtime): render media color grading shaders

Browse Source
This commit is contained in:
ukimsanov
2026-07-06 12:44:43 -07:00
parent e01ec16194
commit 870964b0cf
5 changed files with 969 additions and 129 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/core/src/runtime/colorGrading.test.ts
+253 -5
View File
@@ -4,6 +4,8 @@ import { createColorGradingRuntime, type RuntimeColorGradingApi } from "./colorG
let lastUniform1f: ReturnType<typeof vi.fn> | null = null;
let lastUniform3f: ReturnType<typeof vi.fn> | null = null;
let lastShaderSources: string[] = [];
let texImage2DCalls: unknown[][] = [];
const IDENTITY_2 = `
LUT_3D_SIZE 2
@@ -17,7 +19,7 @@ LUT_3D_SIZE 2
1 1 1
`;
function createMockWebGl(): WebGLRenderingContext {
function createMockWebGl(options: { failMediaUpload?: boolean } = {}): WebGLRenderingContext {
const shader = {};
const program = {};
const texture = {};
@@ -45,11 +47,18 @@ function createMockWebGl(): WebGLRenderingContext {
STATIC_DRAW: 0x88e4,
TEXTURE0: 0x84c0,
TEXTURE1: 0x84c1,
TEXTURE2: 0x84c2,
TEXTURE3: 0x84c3,
FLOAT: 0x1406,
TRIANGLE_STRIP: 0x0005,
UNPACK_FLIP_Y_WEBGL: 0x9240,
FRAMEBUFFER: 0x8d40,
COLOR_ATTACHMENT0: 0x8ce0,
FRAMEBUFFER_COMPLETE: 0x8cd5,
createShader: vi.fn(() => shader),
shaderSource: vi.fn(),
shaderSource: vi.fn((_shader, source: string) => {
lastShaderSources.push(source);
}),
compileShader: vi.fn(),
getShaderParameter: vi.fn(() => true),
getShaderInfoLog: vi.fn(() => ""),
@@ -63,7 +72,17 @@ function createMockWebGl(): WebGLRenderingContext {
createTexture: vi.fn(() => texture),
bindTexture: vi.fn(),
texParameteri: vi.fn(),
texImage2D: vi.fn(),
texImage2D: vi.fn((...args: unknown[]) => {
texImage2DCalls.push(args);
if (options.failMediaUpload && args.length === 6) {
throw new Error("Media cannot be sampled by WebGL");
}
}),
createFramebuffer: vi.fn(() => ({})),
bindFramebuffer: vi.fn(),
framebufferTexture2D: vi.fn(),
checkFramebufferStatus: vi.fn(() => 0x8cd5),
deleteFramebuffer: vi.fn(),
createBuffer: vi.fn(() => buffer),
bindBuffer: vi.fn(),
bufferData: vi.fn(),
@@ -81,6 +100,7 @@ function createMockWebGl(): WebGLRenderingContext {
vertexAttribPointer: vi.fn(),
drawArrays: vi.fn(),
deleteTexture: vi.fn(),
deleteBuffer: vi.fn(),
} as unknown as WebGLRenderingContext;
}
@@ -113,12 +133,12 @@ function makeDrawableVideo(): HTMLVideoElement {
return video;
}
function stubCubeLutFetch(): ReturnType<typeof vi.fn> {
function stubCubeLutFetch(text = IDENTITY_2): ReturnType<typeof vi.fn> {
const fetchMock = vi.fn(() =>
Promise.resolve({
ok: true,
status: 200,
text: () => Promise.resolve(IDENTITY_2),
text: () => Promise.resolve(text),
}),
);
vi.stubGlobal("fetch", fetchMock);
@@ -133,6 +153,8 @@ describe("createColorGradingRuntime", () => {
document.body.innerHTML = "";
lastUniform1f = null;
lastUniform3f = null;
lastShaderSources = [];
texImage2DCalls = [];
getContextSpy = vi
.spyOn(HTMLCanvasElement.prototype, "getContext")
.mockImplementation((type: string) =>
@@ -172,6 +194,7 @@ describe("createColorGradingRuntime", () => {
it("re-hides source media after timeline visibility sync", () => {
const { video, canvas } = startRuntimeWithVideo();
expect(canvas.id).toBe("__hf_color_grading_hero-video");
expect(video.style.getPropertyValue("visibility")).toBe("");
expect(video.style.getPropertyValue("opacity")).toBe("0");
expect(video.style.getPropertyPriority("opacity")).toBe("important");
@@ -258,6 +281,70 @@ describe("createColorGradingRuntime", () => {
expect(video.style.getPropertyPriority("opacity")).toBe("important");
});
it("keeps the canvas visible when producer render-frame injection hides the source video", () => {
const video = makeDrawableVideo();
Object.defineProperty(video, "readyState", {
value: HTMLMediaElement.HAVE_METADATA,
configurable: true,
});
Object.defineProperty(video, "videoWidth", { value: 0, configurable: true });
Object.defineProperty(video, "videoHeight", { value: 0, configurable: true });
document.body.appendChild(video);
runtime = createColorGradingRuntime();
const canvas = document.querySelector<HTMLCanvasElement>("[data-hf-color-grading-canvas]");
if (!canvas) throw new Error("Expected color grading canvas");
expect(canvas.style.display).toBe("none");
const frame = document.createElement("img");
frame.id = "__render_frame_hero-video__";
frame.className = "__render_frame__";
frame.style.visibility = "visible";
frame.style.opacity = "0.75";
Object.defineProperty(frame, "complete", { value: true, configurable: true });
Object.defineProperty(frame, "naturalWidth", { value: 640, configurable: true });
Object.defineProperty(frame, "naturalHeight", { value: 360, configurable: true });
video.parentNode?.insertBefore(frame, canvas);
video.style.setProperty("visibility", "hidden", "important");
runtime.redraw();
expect(canvas.style.display).toBe("block");
expect(canvas.style.visibility).toBe("visible");
expect(canvas.style.opacity).toBe("0.75");
});
it("moves the canvas above producer render-frame images before capture", () => {
const video = makeDrawableVideo();
Object.defineProperty(video, "readyState", {
value: HTMLMediaElement.HAVE_METADATA,
configurable: true,
});
Object.defineProperty(video, "videoWidth", { value: 0, configurable: true });
Object.defineProperty(video, "videoHeight", { value: 0, configurable: true });
document.body.appendChild(video);
runtime = createColorGradingRuntime();
const canvas = document.querySelector<HTMLCanvasElement>("[data-hf-color-grading-canvas]");
if (!canvas) throw new Error("Expected color grading canvas");
const frame = document.createElement("img");
frame.id = "__render_frame_hero-video__";
frame.className = "__render_frame__";
Object.defineProperty(frame, "complete", { value: true, configurable: true });
Object.defineProperty(frame, "naturalWidth", { value: 640, configurable: true });
Object.defineProperty(frame, "naturalHeight", { value: 360, configurable: true });
video.parentNode?.insertBefore(frame, canvas.nextSibling);
expect(video.nextSibling).toBe(canvas);
expect(canvas.nextSibling).toBe(frame);
runtime.redraw();
expect(video.nextSibling).toBe(frame);
expect(frame.nextSibling).toBe(canvas);
});
it("updates before-after compare uniforms without changing the source grading", () => {
const video = makeDrawableVideo();
document.body.appendChild(video);
@@ -279,6 +366,86 @@ describe("createColorGradingRuntime", () => {
);
});
it("passes finishing detail uniforms into the shader", () => {
const video = makeDrawableVideo();
video.setAttribute(
HF_COLOR_GRADING_ATTR,
serializeHfColorGrading({
adjust: { vibrance: 0.35 },
details: {
vignette: 0.4,
vignetteMidpoint: 0.35,
vignetteRoundness: -0.25,
vignetteFeather: 0.8,
grain: 0.2,
grainSize: 0.7,
grainRoughness: 0.3,
},
effects: { blur: 0.3, pixelate: 0.1 },
}),
);
document.body.appendChild(video);
runtime = createColorGradingRuntime();
if (!lastUniform1f) throw new Error("Expected WebGL uniform calls");
expect(lastUniform1f).toHaveBeenCalledWith("u_vibrance", 0.35);
expect(lastUniform1f).toHaveBeenCalledWith("u_vignette", 0.4);
expect(lastUniform1f).toHaveBeenCalledWith("u_vignetteMidpoint", 0.35);
expect(lastUniform1f).toHaveBeenCalledWith("u_vignetteRoundness", -0.25);
expect(lastUniform1f).toHaveBeenCalledWith("u_vignetteFeather", 0.8);
expect(lastUniform1f).toHaveBeenCalledWith("u_grain", 0.2);
expect(lastUniform1f).toHaveBeenCalledWith("u_grainSize", 0.7);
expect(lastUniform1f).toHaveBeenCalledWith("u_grainRoughness", 0.3);
expect(lastUniform1f).toHaveBeenCalledWith("u_grainSeed", expect.any(Number));
expect(lastUniform1f).toHaveBeenCalledWith("u_blur", 0.3);
expect(lastUniform1f).toHaveBeenCalledWith("u_pixelate", 0.1);
});
it("uses the effected media sample as the graded shader input", () => {
const video = makeDrawableVideo();
video.setAttribute(HF_COLOR_GRADING_ATTR, serializeHfColorGrading({ effects: { blur: 0.25 } }));
document.body.appendChild(video);
runtime = createColorGradingRuntime();
const fragment = lastShaderSources.find((source) => source.includes("sampleMedia"));
expect(fragment).toContain("vec4 originalSample = sampleSource(uv);");
expect(fragment).toContain("vec4 sampleColor = sampleMedia(uv);");
expect(fragment).toContain("uniform sampler2D u_blurSource;");
expect(fragment).toContain("floor(clamp(uv, vec2(0.0), vec2(0.999999)) * cells)");
expect(fragment).toContain("vec2 vignetteAspect");
expect(fragment).toContain("float vibranceWeight");
expect(fragment).toContain("float vignettePower");
expect(fragment).toContain("float grainMask");
expect(fragment).toContain("float grainPixelSize");
expect(fragment).toContain("float blackPoint = clamp(u_blacks * 0.18");
expect(fragment).toContain("float whitePoint = clamp(1.0 - u_whites * 0.18");
expect(fragment).toContain("vec3 color = sampleColor.rgb * pow(2.0, u_exposure);");
expect(fragment).not.toContain("sampleSoft");
const blurFragment = lastShaderSources.find((source) =>
source.includes("uniform vec2 u_direction;"),
);
expect(blurFragment).toContain("stepUv * 12.0");
expect(blurFragment).toContain("color.rgb *= color.a;");
expect(blurFragment).toContain("color.rgb /= color.a;");
});
it("renders blur passes at media resolution to avoid blocky high-strength blur", () => {
const video = makeDrawableVideo();
video.setAttribute(HF_COLOR_GRADING_ATTR, serializeHfColorGrading({ effects: { blur: 1 } }));
document.body.appendChild(video);
runtime = createColorGradingRuntime();
expect(texImage2DCalls.some((args) => args[3] === 640 && args[4] === 360)).toBe(true);
expect(
lastUniform1f?.mock.calls.some(
([name, value]) => name === "u_radius" && typeof value === "number" && value > 30,
),
).toBe(true);
});
it("loads cube LUTs and enables LUT uniforms", async () => {
const fetchMock = stubCubeLutFetch();
const origin = window.location.origin;
@@ -305,4 +472,85 @@ describe("createColorGradingRuntime", () => {
expect(lastUniform3f).toHaveBeenCalledWith("u_lutDomainMax", 1, 1, 1);
expect(runtime.getStatus("#hero-video").message).toBe("Shader + LUT active");
});
it("bounds the runtime LUT cache", async () => {
const fetchMock = stubCubeLutFetch();
const origin = window.location.origin;
document.head.innerHTML = `<base href="${origin}/api/projects/demo/preview/">`;
const { video } = startRuntimeWithVideo();
for (let index = 0; index < 17; index += 1) {
runtime?.setGrading(`#${video.id}`, {
lut: { src: `assets/luts/${index}.cube`, intensity: 1 },
});
await flushLutLoad();
}
runtime?.setGrading(`#${video.id}`, {
lut: { src: "assets/luts/0.cube", intensity: 1 },
});
await flushLutLoad();
const firstUrl = `${origin}/api/projects/demo/preview/assets/luts/0.cube`;
expect(fetchMock.mock.calls.filter(([url]) => url === firstUrl)).toHaveLength(2);
});
it("reports unsupported LUT files in runtime status", async () => {
stubCubeLutFetch(`
LUT_1D_SIZE 2
0 0 0
1 1 1
`);
const origin = window.location.origin;
document.head.innerHTML = `<base href="${origin}/api/projects/demo/preview/">`;
const video = makeDrawableVideo();
video.setAttribute(
HF_COLOR_GRADING_ATTR,
serializeHfColorGrading({ lut: { src: "assets/luts/oned.cube", intensity: 1 } }),
);
document.body.appendChild(video);
runtime = createColorGradingRuntime();
await flushLutLoad();
expect(runtime.getStatus("#hero-video").message).toContain(
"LUT error: 1D cube LUTs are not supported yet",
);
});
it("reports media texture upload failures in runtime status", () => {
getContextSpy.mockImplementation((type: string) =>
type === "webgl" ? createMockWebGl({ failMediaUpload: true }) : null,
);
const video = makeDrawableVideo();
document.body.appendChild(video);
runtime = createColorGradingRuntime();
expect(runtime.getStatus("#hero-video")).toEqual({
state: "unavailable",
message: "Media cannot be sampled by WebGL",
});
});
it("falls back to the source media on WebGL context loss and redraws after restore", () => {
const { video, canvas } = startRuntimeWithVideo();
const lost = new Event("webglcontextlost", { cancelable: true });
canvas.dispatchEvent(lost);
expect(lost.defaultPrevented).toBe(true);
expect(canvas.style.display).toBe("none");
expect(video.hasAttribute("data-hf-color-grading-source-hidden")).toBe(false);
expect(video.style.getPropertyValue("opacity")).toBe("");
expect(runtime?.getStatus("#hero-video")).toEqual({
state: "unavailable",
message: "WebGL context lost",
});
canvas.dispatchEvent(new Event("webglcontextrestored"));
expect(runtime?.getStatus("#hero-video").state).toBe("active");
expect(video.hasAttribute("data-hf-color-grading-source-hidden")).toBe(true);
expect(video.style.getPropertyValue("opacity")).toBe("0");
});
});
packages/core/src/runtime/colorGrading.ts
+522 -43
View File
@@ -1,5 +1,6 @@
import {
HF_COLOR_GRADING_ATTR,
HF_COLOR_GRADING_CANVAS_ID_PREFIX,
isHfColorGradingActive,
normalizeHfColorGrading,
normalizeHfColorGradingWithVariables,
@@ -7,7 +8,13 @@ import {
type HfColorGradingTarget,
type NormalizedHfColorGrading,
} from "../colorGrading";
import { packCubeLutToRgba8, parseCubeLut, type CubeLut3D, type CubeLutVec3 } from "../colorLuts";
import {
DEFAULT_MAX_CUBE_LUT_SIZE,
packCubeLutToRgba8,
parseCubeLut,
type CubeLut3D,
type CubeLutVec3,
} from "../colorLuts";
import { copyMediaVisualStyles } from "../inline-scripts/parityContract";
import { swallow } from "./diagnostics";
@@ -34,12 +41,15 @@ interface ProgramInfo {
program: WebGLProgram;
texture: WebGLTexture;
lutTexture: WebGLTexture;
quad: WebGLBuffer;
position: number;
source: WebGLUniformLocation | null;
blurSource: WebGLUniformLocation | null;
lut: WebGLUniformLocation | null;
resolution: WebGLUniformLocation | null;
uvScale: WebGLUniformLocation | null;
uvOffset: WebGLUniformLocation | null;
blurReady: WebGLUniformLocation | null;
lutEnabled: WebGLUniformLocation | null;
lutSize: WebGLUniformLocation | null;
lutTextureSize: WebGLUniformLocation | null;
@@ -54,7 +64,18 @@ interface ProgramInfo {
blacks: WebGLUniformLocation | null;
temperature: WebGLUniformLocation | null;
tint: WebGLUniformLocation | null;
vibrance: WebGLUniformLocation | null;
saturation: WebGLUniformLocation | null;
vignette: WebGLUniformLocation | null;
vignetteMidpoint: WebGLUniformLocation | null;
vignetteRoundness: WebGLUniformLocation | null;
vignetteFeather: WebGLUniformLocation | null;
grain: WebGLUniformLocation | null;
grainSize: WebGLUniformLocation | null;
grainRoughness: WebGLUniformLocation | null;
grainSeed: WebGLUniformLocation | null;
blur: WebGLUniformLocation | null;
pixelate: WebGLUniformLocation | null;
intensity: WebGLUniformLocation | null;
compareEnabled: WebGLUniformLocation | null;
comparePosition: WebGLUniformLocation | null;
@@ -62,6 +83,29 @@ interface ProgramInfo {
compareLineWidth: WebGLUniformLocation | null;
}
interface BlurProgramInfo {
program: WebGLProgram;
quad: WebGLBuffer;
position: number;
source: WebGLUniformLocation | null;
resolution: WebGLUniformLocation | null;
direction: WebGLUniformLocation | null;
radius: WebGLUniformLocation | null;
}
interface RenderTarget {
texture: WebGLTexture;
framebuffer: WebGLFramebuffer;
width: number;
height: number;
}
interface EffectTargets {
blurProgram: BlurProgramInfo;
scratch: RenderTarget;
blur: RenderTarget;
}
interface RuntimeColorGradingCompareState {
enabled: boolean;
position: number;
@@ -79,6 +123,9 @@ interface ColorGradingEntry {
lut: RuntimeLutTexture | null;
lutLoadingSrc: string | null;
lutError: string | null;
drawError: string | null;
effectTargets: EffectTargets | null;
effectError: string | null;
source: EntrySource;
animationFrame: number | null;
videoFrameHandle: number | null;
@@ -92,6 +139,8 @@ interface ColorGradingEntry {
sourceOpacityForCanvas: string;
sourceVisibleForCanvas: boolean;
hasDrawn: boolean;
contextLost: boolean;
grainSeed: number;
destroyed: boolean;
}
@@ -133,7 +182,6 @@ type WindowWithColorGrading = Window & {
interface RuntimeLutTexture {
src: string;
title: string | null;
size: number;
domainMin: CubeLutVec3;
domainMax: CubeLutVec3;
@@ -150,7 +198,8 @@ const LUT_CACHE = new Map<string, LutCacheEntry>();
const COLOR_GRADING_CANVAS_ATTR = "data-hf-color-grading-canvas";
const COLOR_GRADING_SOURCE_HIDDEN_ATTR = "data-hf-color-grading-source-hidden";
const COLOR_GRADING_CANVAS_CLASS = "__hf_color_grading_canvas__";
const MAX_LUT_SIZE = 64;
// Map insertion order gives us simple FIFO eviction for authoring sessions that cycle LUTs.
const MAX_LUT_CACHE_ENTRIES = 16;
const DEFAULT_COMPARE: RuntimeColorGradingCompareState = {
enabled: false,
position: 0.5,
@@ -195,10 +244,12 @@ const FRAGMENT_SHADER = [
"#endif",
"varying vec2 v_uv;",
"uniform sampler2D u_source;",
"uniform sampler2D u_blurSource;",
"uniform sampler2D u_lut;",
"uniform vec2 u_resolution;",
"uniform vec2 u_uvScale;",
"uniform vec2 u_uvOffset;",
"uniform float u_blurReady;",
"uniform float u_lutEnabled;",
"uniform float u_lutSize;",
"uniform vec2 u_lutTextureSize;",
@@ -213,13 +264,44 @@ const FRAGMENT_SHADER = [
"uniform float u_blacks;",
"uniform float u_temperature;",
"uniform float u_tint;",
"uniform float u_vibrance;",
"uniform float u_saturation;",
"uniform float u_vignette;",
"uniform float u_vignetteMidpoint;",
"uniform float u_vignetteRoundness;",
"uniform float u_vignetteFeather;",
"uniform float u_grain;",
"uniform float u_grainSize;",
"uniform float u_grainRoughness;",
"uniform float u_grainSeed;",
"uniform float u_blur;",
"uniform float u_pixelate;",
"uniform float u_intensity;",
"uniform float u_compareEnabled;",
"uniform float u_comparePosition;",
"uniform float u_compareSoftness;",
"uniform float u_compareLineWidth;",
"float lumaOf(vec3 c){ return dot(c, vec3(0.2126, 0.7152, 0.0722)); }",
"float grainHash(vec2 p){ return fract(sin(dot(p, vec2(127.1, 311.7))) * 43758.5453123); }",
"float colorSaturation(vec3 c){ return max(max(c.r, c.g), c.b) - min(min(c.r, c.g), c.b); }",
"vec2 clampUv(vec2 uv){ return clamp(uv, vec2(0.0), vec2(1.0)); }",
"vec4 sampleSource(vec2 uv){ return texture2D(u_source, clampUv(uv)); }",
"vec4 sampleBlur(vec2 uv){ return texture2D(u_blurSource, clampUv(uv)); }",
"vec4 sampleMedia(vec2 uv){",
" float pixel = clamp(u_pixelate, 0.0, 1.0);",
" vec2 sampleUv = uv;",
" if (pixel > 0.0) {",
" float blockSize = mix(1.0, 48.0, pixel);",
" vec2 cells = max(u_resolution / blockSize, vec2(1.0));",
" sampleUv = (floor(clamp(uv, vec2(0.0), vec2(0.999999)) * cells) + 0.5) / cells;",
" }",
" vec4 base = sampleSource(sampleUv);",
" float blur = clamp(u_blur, 0.0, 1.0);",
" if (blur > 0.0 && u_blurReady > 0.5) {",
" base = mix(base, sampleBlur(sampleUv), blur);",
" }",
" return base;",
"}",
"vec3 sampleLut(float r, float g, float b){",
" float size = max(u_lutSize, 2.0);",
" float x = (r + b * size + 0.5) / max(u_lutTextureSize.x, 1.0);",
@@ -257,24 +339,52 @@ const FRAGMENT_SHADER = [
" gl_FragColor = vec4(0.0);",
" return;",
" }",
" vec4 sampleColor = texture2D(u_source, uv);",
" vec3 original = sampleColor.rgb;",
" vec3 color = original * pow(2.0, u_exposure);",
" vec4 originalSample = sampleSource(uv);",
" vec4 sampleColor = sampleMedia(uv);",
" vec3 original = originalSample.rgb;",
" vec3 color = sampleColor.rgb * pow(2.0, u_exposure);",
" float y = lumaOf(color);",
" float shadowMask = 1.0 - smoothstep(0.0, 0.65, y);",
" float highlightMask = smoothstep(0.35, 1.0, y);",
" color += u_shadows * 0.35 * shadowMask;",
" color += u_highlights * 0.35 * highlightMask;",
" color += u_blacks * 0.25 * (1.0 - smoothstep(0.0, 0.35, y));",
" color += u_whites * 0.25 * smoothstep(0.65, 1.0, y);",
" float blackPoint = clamp(u_blacks * 0.18, -0.18, 0.18);",
" float whitePoint = clamp(1.0 - u_whites * 0.18, 0.82, 1.18);",
" color = (color - blackPoint) / max(whitePoint - blackPoint, 0.2);",
" color.r += u_temperature * 0.08 + u_tint * 0.04;",
" color.b -= u_temperature * 0.08 - u_tint * 0.04;",
" color.g -= u_tint * 0.08;",
" color = (color - 0.5) * max(0.0, 1.0 + u_contrast) + 0.5;",
" float satLuma = lumaOf(color);",
" float currentSat = clamp(colorSaturation(color), 0.0, 1.0);",
" float skinLike = smoothstep(0.02, 0.18, color.r - color.g) * smoothstep(0.0, 0.16, color.g - color.b) * smoothstep(0.18, 0.82, satLuma);",
" float vibranceWeight = (1.0 - currentSat * 0.72) * mix(1.0, 0.55, skinLike);",
" color = mix(vec3(satLuma), color, max(0.0, 1.0 + u_vibrance * vibranceWeight));",
" color = mix(vec3(satLuma), color, max(0.0, 1.0 + u_saturation));",
" color = clamp(color, 0.0, 1.0);",
" color = clamp(applyLut(color), 0.0, 1.0);",
" vec2 vignetteAspect = u_resolution.x > u_resolution.y",
" ? vec2(u_resolution.x / max(u_resolution.y, 1.0), 1.0)",
" : vec2(1.0, u_resolution.y / max(u_resolution.x, 1.0));",
" vec2 vignetteUv = abs((v_uv - vec2(0.5)) * 2.0) * vignetteAspect;",
" float vignettePower = mix(8.0, 1.8, clamp(u_vignetteRoundness * 0.5 + 0.5, 0.0, 1.0));",
" float vignetteDistance = pow(pow(vignetteUv.x, vignettePower) + pow(vignetteUv.y, vignettePower), 1.0 / vignettePower);",
" float vignetteMidpoint = mix(0.22, 1.08, clamp(u_vignetteMidpoint, 0.0, 1.0));",
" float vignetteFeather = mix(0.08, 0.72, clamp(u_vignetteFeather, 0.0, 1.0));",
" float vignetteMask = smoothstep(vignetteMidpoint, vignetteMidpoint + vignetteFeather, vignetteDistance);",
" color *= 1.0 - vignetteMask * clamp(u_vignette, 0.0, 1.0) * 0.75;",
" float grainAmount = clamp(u_grain, 0.0, 1.0);",
" if (grainAmount > 0.0) {",
" float grainPixelSize = mix(1.0, 6.0, clamp(u_grainSize, 0.0, 1.0));",
" vec2 grainCoord = floor(gl_FragCoord.xy / grainPixelSize) + vec2(u_grainSeed, u_grainSeed * 1.37);",
" float grainBase = grainHash(grainCoord) - grainHash(grainCoord + vec2(19.19, 73.31));",
" float grainFine = grainHash(gl_FragCoord.xy + vec2(u_grainSeed * 2.11, u_grainSeed * 0.71)) - 0.5;",
" float grain = mix(grainBase * 0.7, grainBase + grainFine * 0.35, clamp(u_grainRoughness, 0.0, 1.0));",
" float grainLuma = lumaOf(color);",
" float grainMask = smoothstep(0.02, 0.55, grainLuma) * (1.0 - smoothstep(0.88, 1.0, grainLuma));",
" color += grain * grainAmount * mix(0.025, 0.08, grainMask);",
" }",
" color = clamp(color, 0.0, 1.0);",
" vec3 graded = mix(original, color, u_intensity);",
" if (u_compareEnabled > 0.5) {",
" float pos = clamp(u_comparePosition, 0.0, 1.0);",
@@ -293,6 +403,54 @@ const FRAGMENT_SHADER = [
"}",
].join("\n");
const BLUR_FRAGMENT_SHADER = [
"#ifdef GL_FRAGMENT_PRECISION_HIGH",
"precision highp float;",
"#else",
"precision mediump float;",
"#endif",
"varying vec2 v_uv;",
"uniform sampler2D u_source;",
"uniform vec2 u_resolution;",
"uniform vec2 u_direction;",
"uniform float u_radius;",
"vec4 readSource(vec2 uv){",
" vec4 color = texture2D(u_source, clamp(uv, vec2(0.0), vec2(1.0)));",
" color.rgb *= color.a;",
" return color;",
"}",
"void main(){",
" vec2 stepUv = u_direction * max(u_radius, 0.0) / max(u_resolution, vec2(1.0)) / 12.0;",
" vec4 color = readSource(v_uv) * 0.08077993;",
" color += readSource(v_uv + stepUv * 1.0) * 0.07918038;",
" color += readSource(v_uv - stepUv * 1.0) * 0.07918038;",
" color += readSource(v_uv + stepUv * 2.0) * 0.07456928;",
" color += readSource(v_uv - stepUv * 2.0) * 0.07456928;",
" color += readSource(v_uv + stepUv * 3.0) * 0.06747307;",
" color += readSource(v_uv - stepUv * 3.0) * 0.06747307;",
" color += readSource(v_uv + stepUv * 4.0) * 0.05865827;",
" color += readSource(v_uv - stepUv * 4.0) * 0.05865827;",
" color += readSource(v_uv + stepUv * 5.0) * 0.04899551;",
" color += readSource(v_uv - stepUv * 5.0) * 0.04899551;",
" color += readSource(v_uv + stepUv * 6.0) * 0.03931982;",
" color += readSource(v_uv - stepUv * 6.0) * 0.03931982;",
" color += readSource(v_uv + stepUv * 7.0) * 0.03031761;",
" color += readSource(v_uv - stepUv * 7.0) * 0.03031761;",
" color += readSource(v_uv + stepUv * 8.0) * 0.02245983;",
" color += readSource(v_uv - stepUv * 8.0) * 0.02245983;",
" color += readSource(v_uv + stepUv * 9.0) * 0.01598624;",
" color += readSource(v_uv - stepUv * 9.0) * 0.01598624;",
" color += readSource(v_uv + stepUv * 10.0) * 0.01093238;",
" color += readSource(v_uv - stepUv * 10.0) * 0.01093238;",
" color += readSource(v_uv + stepUv * 11.0) * 0.00718308;",
" color += readSource(v_uv - stepUv * 11.0) * 0.00718308;",
" color += readSource(v_uv + stepUv * 12.0) * 0.00453456;",
" color += readSource(v_uv - stepUv * 12.0) * 0.00453456;",
" if (color.a > 0.0001) color.rgb /= color.a;",
" gl_FragColor = color;",
"}",
].join("\n");
function isColorGradingMediaElement(value: Element): value is ColorGradingMediaElement {
return value instanceof HTMLVideoElement || value instanceof HTMLImageElement;
}
@@ -314,9 +472,12 @@ function compileShader(
return shader;
}
function createProgram(gl: WebGLRenderingContext): WebGLProgram | null {
function createProgram(
gl: WebGLRenderingContext,
fragmentSource = FRAGMENT_SHADER,
): WebGLProgram | null {
const vertex = compileShader(gl, VERTEX_SHADER, gl.VERTEX_SHADER);
const fragment = compileShader(gl, FRAGMENT_SHADER, gl.FRAGMENT_SHADER);
const fragment = compileShader(gl, fragmentSource, gl.FRAGMENT_SHADER);
if (!vertex || !fragment) {
if (vertex) gl.deleteShader(vertex);
if (fragment) gl.deleteShader(fragment);
@@ -349,6 +510,56 @@ function createTexture(gl: WebGLRenderingContext, filter = gl.LINEAR): WebGLText
return texture;
}
function createBlurProgramInfo(
gl: WebGLRenderingContext,
quad: WebGLBuffer,
): BlurProgramInfo | null {
const program = createProgram(gl, BLUR_FRAGMENT_SHADER);
if (!program) return null;
return {
program,
quad,
position: gl.getAttribLocation(program, "a_pos"),
source: gl.getUniformLocation(program, "u_source"),
resolution: gl.getUniformLocation(program, "u_resolution"),
direction: gl.getUniformLocation(program, "u_direction"),
radius: gl.getUniformLocation(program, "u_radius"),
};
}
function createRenderTarget(gl: WebGLRenderingContext): RenderTarget | null {
const texture = createTexture(gl);
const framebuffer = gl.createFramebuffer();
if (!texture || !framebuffer) {
if (texture) gl.deleteTexture(texture);
if (framebuffer) gl.deleteFramebuffer(framebuffer);
return null;
}
gl.bindFramebuffer(gl.FRAMEBUFFER, framebuffer);
gl.framebufferTexture2D(gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.TEXTURE_2D, texture, 0);
const status = gl.checkFramebufferStatus(gl.FRAMEBUFFER);
gl.bindFramebuffer(gl.FRAMEBUFFER, null);
if (status !== gl.FRAMEBUFFER_COMPLETE) {
gl.deleteTexture(texture);
gl.deleteFramebuffer(framebuffer);
return null;
}
return { texture, framebuffer, width: 1, height: 1 };
}
function resizeRenderTarget(
gl: WebGLRenderingContext,
target: RenderTarget,
width: number,
height: number,
): void {
if (target.width === width && target.height === height) return;
target.width = width;
target.height = height;
gl.bindTexture(gl.TEXTURE_2D, target.texture);
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, width, height, 0, gl.RGBA, gl.UNSIGNED_BYTE, null);
}
function createProgramInfo(canvas: HTMLCanvasElement): {
gl: WebGLRenderingContext;
program: ProgramInfo;
@@ -384,12 +595,15 @@ function createProgramInfo(canvas: HTMLCanvasElement): {
program,
texture,
lutTexture,
quad,
position: gl.getAttribLocation(program, "a_pos"),
source: gl.getUniformLocation(program, "u_source"),
blurSource: gl.getUniformLocation(program, "u_blurSource"),
lut: gl.getUniformLocation(program, "u_lut"),
resolution: gl.getUniformLocation(program, "u_resolution"),
uvScale: gl.getUniformLocation(program, "u_uvScale"),
uvOffset: gl.getUniformLocation(program, "u_uvOffset"),
blurReady: gl.getUniformLocation(program, "u_blurReady"),
lutEnabled: gl.getUniformLocation(program, "u_lutEnabled"),
lutSize: gl.getUniformLocation(program, "u_lutSize"),
lutTextureSize: gl.getUniformLocation(program, "u_lutTextureSize"),
@@ -404,7 +618,18 @@ function createProgramInfo(canvas: HTMLCanvasElement): {
blacks: gl.getUniformLocation(program, "u_blacks"),
temperature: gl.getUniformLocation(program, "u_temperature"),
tint: gl.getUniformLocation(program, "u_tint"),
vibrance: gl.getUniformLocation(program, "u_vibrance"),
saturation: gl.getUniformLocation(program, "u_saturation"),
vignette: gl.getUniformLocation(program, "u_vignette"),
vignetteMidpoint: gl.getUniformLocation(program, "u_vignetteMidpoint"),
vignetteRoundness: gl.getUniformLocation(program, "u_vignetteRoundness"),
vignetteFeather: gl.getUniformLocation(program, "u_vignetteFeather"),
grain: gl.getUniformLocation(program, "u_grain"),
grainSize: gl.getUniformLocation(program, "u_grainSize"),
grainRoughness: gl.getUniformLocation(program, "u_grainRoughness"),
grainSeed: gl.getUniformLocation(program, "u_grainSeed"),
blur: gl.getUniformLocation(program, "u_blur"),
pixelate: gl.getUniformLocation(program, "u_pixelate"),
intensity: gl.getUniformLocation(program, "u_intensity"),
compareEnabled: gl.getUniformLocation(program, "u_compareEnabled"),
comparePosition: gl.getUniformLocation(program, "u_comparePosition"),
@@ -452,6 +677,24 @@ function errorMessage(err: unknown): string {
return err instanceof Error ? err.message : "LUT failed to load";
}
function hashStringSeed(value: string): number {
let hash = 2166136261;
for (let index = 0; index < value.length; index += 1) {
hash ^= value.charCodeAt(index);
hash = Math.imul(hash, 16777619);
}
return (hash >>> 0) % 10000;
}
function seedForElement(element: ColorGradingMediaElement): number {
const key =
element.id ||
element.currentSrc ||
element.getAttribute("src") ||
`${element.tagName}:${Array.prototype.indexOf.call(element.parentNode?.children ?? [], element)}`;
return hashStringSeed(key);
}
function getCubeLut(src: string): LutCacheEntry {
const resolved = resolveLutUrl(src);
if ("error" in resolved) return { state: "error", message: resolved.error };
@@ -463,13 +706,26 @@ function getCubeLut(src: string): LutCacheEntry {
if (!response.ok) throw new Error(`Failed to load LUT (${response.status})`);
return response.text();
})
.then((text) => parseCubeLut(text, { maxSize: MAX_LUT_SIZE }));
.then((text) => parseCubeLut(text, { maxSize: DEFAULT_MAX_CUBE_LUT_SIZE }));
const pending: LutCacheEntry = { state: "pending", promise };
while (LUT_CACHE.size >= MAX_LUT_CACHE_ENTRIES) {
const oldest = LUT_CACHE.keys().next().value;
if (!oldest) break;
LUT_CACHE.delete(oldest);
}
LUT_CACHE.set(resolved.href, pending);
promise.then(
(lut) => LUT_CACHE.set(resolved.href, { state: "ready", lut }),
(err) => LUT_CACHE.set(resolved.href, { state: "error", message: errorMessage(err) }),
(lut) => {
if (LUT_CACHE.get(resolved.href) === pending) {
LUT_CACHE.set(resolved.href, { state: "ready", lut });
}
},
(err) => {
if (LUT_CACHE.get(resolved.href) === pending) {
LUT_CACHE.set(resolved.href, { state: "error", message: errorMessage(err) });
}
},
);
return pending;
}
@@ -483,7 +739,7 @@ function uploadEntryLut(
const packed = packCubeLutToRgba8(lut);
const { gl, program } = entry;
try {
gl.activeTexture(gl.TEXTURE1);
gl.activeTexture(gl.TEXTURE2);
gl.bindTexture(gl.TEXTURE_2D, program.lutTexture);
gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, false);
gl.texImage2D(
@@ -499,7 +755,6 @@ function uploadEntryLut(
);
entry.lut = {
src,
title: lut.title,
size: lut.size,
domainMin: lut.domainMin,
domainMax: lut.domainMax,
@@ -518,6 +773,149 @@ function uploadEntryLut(
}
}
function destroyRenderTarget(gl: WebGLRenderingContext, target: RenderTarget): void {
gl.deleteTexture(target.texture);
gl.deleteFramebuffer(target.framebuffer);
}
function destroyEffectTargets(entry: ColorGradingEntry): void {
const targets = entry.effectTargets;
if (!targets) return;
entry.gl.deleteProgram(targets.blurProgram.program);
destroyRenderTarget(entry.gl, targets.scratch);
destroyRenderTarget(entry.gl, targets.blur);
entry.effectTargets = null;
}
function destroyProgramResources(entry: ColorGradingEntry): void {
destroyEffectTargets(entry);
entry.gl.deleteTexture(entry.program.texture);
entry.gl.deleteTexture(entry.program.lutTexture);
entry.gl.deleteBuffer(entry.program.quad);
entry.gl.deleteProgram(entry.program.program);
}
function replaceProgramResources(entry: ColorGradingEntry): boolean {
const created = createProgramInfo(entry.canvas);
if (!created) return false;
destroyProgramResources(entry);
entry.gl = created.gl;
entry.program = created.program;
entry.lut = null;
entry.lutLoadingSrc = null;
entry.lutError = null;
entry.effectError = null;
return true;
}
function restoreSourceElement(entry: ColorGradingEntry): void {
if (!entry.sourceHidden) return;
entry.element.removeAttribute(COLOR_GRADING_SOURCE_HIDDEN_ATTR);
const opacity = entry.element.style.getPropertyValue("opacity");
const priority = entry.element.style.getPropertyPriority("opacity");
if (opacity === "0" && priority === "important") {
if (entry.sourceInlineOpacity === null) {
entry.element.style.removeProperty("opacity");
} else {
entry.element.style.setProperty(
"opacity",
entry.sourceInlineOpacity,
entry.sourceInlineOpacityPriority,
);
}
}
entry.sourceHidden = false;
}
function ensureEffectTargets(entry: ColorGradingEntry): EffectTargets | null {
if (entry.effectTargets) return entry.effectTargets;
const { gl } = entry;
const blurProgram = createBlurProgramInfo(gl, entry.program.quad);
const scratch = createRenderTarget(gl);
const blur = createRenderTarget(gl);
if (!blurProgram || !scratch || !blur) {
if (blurProgram) gl.deleteProgram(blurProgram.program);
if (scratch) destroyRenderTarget(gl, scratch);
if (blur) destroyRenderTarget(gl, blur);
entry.effectError = "Framebuffer effects unavailable";
return null;
}
entry.effectError = null;
entry.effectTargets = { blurProgram, scratch, blur };
return entry.effectTargets;
}
function resizeEffectTargetPair(
gl: WebGLRenderingContext,
scratch: RenderTarget,
output: RenderTarget,
width: number,
height: number,
): { width: number; height: number } {
const targetWidth = Math.max(1, Math.ceil(width));
const targetHeight = Math.max(1, Math.ceil(height));
resizeRenderTarget(gl, scratch, targetWidth, targetHeight);
resizeRenderTarget(gl, output, targetWidth, targetHeight);
return { width: targetWidth, height: targetHeight };
}
function renderBlurPass(
gl: WebGLRenderingContext,
program: BlurProgramInfo,
input: WebGLTexture,
output: RenderTarget,
layout: { width: number; height: number },
direction: { x: number; y: number },
radius: number,
): void {
gl.bindFramebuffer(gl.FRAMEBUFFER, output.framebuffer);
gl.viewport(0, 0, layout.width, layout.height);
gl.useProgram(program.program);
gl.activeTexture(gl.TEXTURE0);
gl.bindTexture(gl.TEXTURE_2D, input);
gl.uniform1i(program.source, 0);
gl.uniform2f(program.resolution, layout.width, layout.height);
gl.uniform2f(program.direction, direction.x, direction.y);
gl.uniform1f(program.radius, radius);
gl.bindBuffer(gl.ARRAY_BUFFER, program.quad);
gl.enableVertexAttribArray(program.position);
gl.vertexAttribPointer(program.position, 2, gl.FLOAT, false, 0, 0);
gl.drawArrays(gl.TRIANGLE_STRIP, 0, 4);
}
function renderGaussianTexture(
gl: WebGLRenderingContext,
targets: EffectTargets,
input: WebGLTexture,
output: RenderTarget,
layout: { width: number; height: number },
radius: number,
iterations: number,
): void {
let source = input;
for (let pass = 0; pass < Math.max(1, Math.floor(iterations)); pass++) {
renderBlurPass(
gl,
targets.blurProgram,
source,
targets.scratch,
layout,
{ x: 1, y: 0 },
radius,
);
renderBlurPass(
gl,
targets.blurProgram,
targets.scratch.texture,
output,
layout,
{ x: 0, y: 1 },
radius,
);
source = output.texture;
}
}
// fallow-ignore-next-line complexity
function ensureEntryLut(entry: ColorGradingEntry): RuntimeLutTexture | null {
const src = entry.grading.lut?.src.trim() ?? "";
@@ -638,6 +1036,16 @@ function findRenderFrameImage(video: HTMLVideoElement): HTMLImageElement | null
return frame instanceof HTMLImageElement && isDrawableSource(frame) ? frame : null;
}
function isRenderFrameImage(source: TexImageSource): source is HTMLImageElement {
return source instanceof HTMLImageElement && source.classList.contains("__render_frame__");
}
function keepCanvasAboveSource(entry: ColorGradingEntry, source: HTMLImageElement): void {
if (source.parentNode && source.nextSibling !== entry.canvas) {
source.parentNode.insertBefore(entry.canvas, source.nextSibling);
}
}
function getDrawableSource(element: ColorGradingMediaElement): TexImageSource | null {
if (element instanceof HTMLVideoElement) {
const renderFrame = findRenderFrameImage(element);
@@ -782,15 +1190,19 @@ function applyUniforms(
program: ProgramInfo,
grading: NormalizedHfColorGrading,
lut: RuntimeLutTexture | null,
blurReady: boolean,
compare: RuntimeColorGradingCompareState,
layout: { width: number; height: number },
uv: { scaleX: number; scaleY: number; offsetX: number; offsetY: number },
grainSeed: number,
): void {
gl.uniform1i(program.source, 0);
gl.uniform1i(program.lut, 1);
gl.uniform1i(program.blurSource, 1);
gl.uniform1i(program.lut, 2);
gl.uniform2f(program.resolution, layout.width, layout.height);
gl.uniform2f(program.uvScale, uv.scaleX, uv.scaleY);
gl.uniform2f(program.uvOffset, uv.offsetX, uv.offsetY);
gl.uniform1f(program.blurReady, blurReady ? 1 : 0);
gl.uniform1f(program.lutEnabled, lut ? 1 : 0);
gl.uniform1f(program.lutSize, lut?.size ?? 2);
gl.uniform2f(program.lutTextureSize, lut?.textureWidth ?? 1, lut?.textureHeight ?? 1);
@@ -815,7 +1227,18 @@ function applyUniforms(
gl.uniform1f(program.blacks, grading.adjust.blacks);
gl.uniform1f(program.temperature, grading.adjust.temperature);
gl.uniform1f(program.tint, grading.adjust.tint);
gl.uniform1f(program.vibrance, grading.adjust.vibrance);
gl.uniform1f(program.saturation, grading.adjust.saturation);
gl.uniform1f(program.vignette, grading.details.vignette);
gl.uniform1f(program.vignetteMidpoint, grading.details.vignetteMidpoint);
gl.uniform1f(program.vignetteRoundness, grading.details.vignetteRoundness);
gl.uniform1f(program.vignetteFeather, grading.details.vignetteFeather);
gl.uniform1f(program.grain, grading.details.grain);
gl.uniform1f(program.grainSize, grading.details.grainSize);
gl.uniform1f(program.grainRoughness, grading.details.grainRoughness);
gl.uniform1f(program.grainSeed, grainSeed);
gl.uniform1f(program.blur, grading.effects.blur);
gl.uniform1f(program.pixelate, grading.effects.pixelate);
gl.uniform1f(program.intensity, grading.intensity);
gl.uniform1f(program.compareEnabled, compare.enabled ? 1 : 0);
gl.uniform1f(program.comparePosition, compare.position);
@@ -835,7 +1258,7 @@ function hideSourceElement(entry: ColorGradingEntry): void {
// fallow-ignore-next-line complexity
function drawEntry(entry: ColorGradingEntry): boolean {
if (entry.destroyed) return false;
if (entry.destroyed || entry.contextLost) return false;
const source = getDrawableSource(entry.element);
if (!source) {
if (!entry.hasDrawn) entry.canvas.style.display = "none";
@@ -849,11 +1272,13 @@ function drawEntry(entry: ColorGradingEntry): boolean {
const hiddenByColorGrading =
entry.sourceHidden && sourceOpacity === "0" && sourceOpacityPriority === "important";
const sourceVisibility = entry.element.style.getPropertyValue("visibility");
if (!hiddenByColorGrading) {
const computed = window.getComputedStyle(entry.element);
const injectedFrameSource = isRenderFrameImage(source);
if (injectedFrameSource) keepCanvasAboveSource(entry, source);
if (injectedFrameSource || !hiddenByColorGrading) {
const computed = window.getComputedStyle(injectedFrameSource ? source : entry.element);
entry.sourceOpacityForCanvas = computed.opacity || "1";
entry.sourceVisibleForCanvas =
sourceVisibility !== "hidden" && computed.visibility !== "hidden";
(injectedFrameSource || sourceVisibility !== "hidden") && computed.visibility !== "hidden";
}
const layout = updateCanvasLayout(entry, styleSource);
if (!layout) return false;
@@ -870,24 +1295,65 @@ function drawEntry(entry: ColorGradingEntry): boolean {
const { gl, program } = entry;
try {
const lut = ensureEntryLut(entry);
gl.viewport(0, 0, layout.width, layout.height);
gl.useProgram(program.program);
gl.activeTexture(gl.TEXTURE0);
gl.bindTexture(gl.TEXTURE_2D, program.texture);
// Browser media elements are top-left oriented; WebGL texture coordinates
// are bottom-left oriented unless the upload is flipped.
gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, true);
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, source);
let blurReady = false;
const blurAmount = Math.min(1, Math.max(0, entry.grading.effects.blur));
if (blurAmount > 0) {
// If framebuffer setup fails, keep the non-blur shader path alive and surface status.
const targets = ensureEffectTargets(entry);
if (targets) {
const blurLayout = resizeEffectTargetPair(
gl,
targets.scratch,
targets.blur,
layout.width,
layout.height,
);
renderGaussianTexture(
gl,
targets,
program.texture,
targets.blur,
blurLayout,
0.75 + Math.pow(blurAmount, 1.35) * 32,
blurAmount > 0.55 ? 3 : 2,
);
blurReady = true;
}
} else {
entry.effectError = null;
if (entry.effectTargets) destroyEffectTargets(entry);
}
gl.bindFramebuffer(gl.FRAMEBUFFER, null);
gl.viewport(0, 0, layout.width, layout.height);
gl.useProgram(program.program);
gl.activeTexture(gl.TEXTURE0);
gl.bindTexture(gl.TEXTURE_2D, program.texture);
gl.activeTexture(gl.TEXTURE1);
gl.bindTexture(gl.TEXTURE_2D, entry.effectTargets?.blur.texture ?? program.texture);
gl.activeTexture(gl.TEXTURE2);
gl.bindTexture(gl.TEXTURE_2D, program.lutTexture);
applyUniforms(gl, program, entry.grading, lut, entry.compare, layout, uv);
const grainSeed =
entry.grainSeed +
(entry.element instanceof HTMLVideoElement ? Math.floor(entry.element.currentTime * 60) : 0);
applyUniforms(gl, program, entry.grading, lut, blurReady, entry.compare, layout, uv, grainSeed);
gl.bindBuffer(gl.ARRAY_BUFFER, program.quad);
gl.enableVertexAttribArray(program.position);
gl.vertexAttribPointer(program.position, 2, gl.FLOAT, false, 0, 0);
gl.drawArrays(gl.TRIANGLE_STRIP, 0, 4);
hideSourceElement(entry);
entry.hasDrawn = true;
entry.drawError = null;
return true;
} catch (err) {
entry.drawError = err instanceof Error ? err.message : "Shader draw failed";
swallow("runtime.colorGrading.drawEntry", err);
return false;
}
@@ -949,6 +1415,24 @@ function installEntryListeners(entry: ColorGradingEntry): void {
addListener(entry, entry.element, "play", () => scheduleVideoDraw(entry));
addListener(entry, entry.element, "pause", redraw);
}
addListener(entry, entry.canvas, "webglcontextlost", (event) => {
event.preventDefault();
entry.contextLost = true;
entry.drawError = "WebGL context lost";
entry.canvas.style.display = "none";
restoreSourceElement(entry);
});
addListener(entry, entry.canvas, "webglcontextrestored", () => {
entry.contextLost = false;
if (!replaceProgramResources(entry)) {
entry.contextLost = true;
entry.drawError = "WebGL context restore failed";
restoreSourceElement(entry);
return;
}
entry.drawError = null;
drawEntry(entry);
});
if (typeof ResizeObserver !== "undefined") {
entry.resizeObserver = new ResizeObserver(redraw);
entry.resizeObserver.observe(entry.element);
@@ -963,25 +1447,8 @@ function destroyEntry(entry: ColorGradingEntry): void {
for (const cleanup of entry.cleanup) cleanup();
entry.cleanup.length = 0;
entry.canvas.remove();
entry.gl.deleteTexture(entry.program.texture);
entry.gl.deleteTexture(entry.program.lutTexture);
entry.gl.deleteProgram(entry.program.program);
if (entry.sourceHidden) {
entry.element.removeAttribute(COLOR_GRADING_SOURCE_HIDDEN_ATTR);
const opacity = entry.element.style.getPropertyValue("opacity");
const priority = entry.element.style.getPropertyPriority("opacity");
if (opacity === "0" && priority === "important") {
if (entry.sourceInlineOpacity === null) {
entry.element.style.removeProperty("opacity");
} else {
entry.element.style.setProperty(
"opacity",
entry.sourceInlineOpacity,
entry.sourceInlineOpacityPriority,
);
}
}
}
destroyProgramResources(entry);
restoreSourceElement(entry);
if (entry.touchedParent) {
if (entry.parentInlinePosition === null) {
entry.touchedParent.style.removeProperty("position");
@@ -993,6 +1460,7 @@ function destroyEntry(entry: ColorGradingEntry): void {
function makeCanvas(element: ColorGradingMediaElement): HTMLCanvasElement {
const canvas = document.createElement("canvas");
if (element.id) canvas.id = `${HF_COLOR_GRADING_CANVAS_ID_PREFIX}${element.id}`;
canvas.className = COLOR_GRADING_CANVAS_CLASS;
canvas.setAttribute(COLOR_GRADING_CANVAS_ATTR, "true");
canvas.setAttribute("data-hyperframes-ignore", "");
@@ -1040,6 +1508,9 @@ export function createColorGradingRuntime(): RuntimeColorGradingApi {
lut: null,
lutLoadingSrc: null,
lutError: null,
drawError: null,
effectTargets: null,
effectError: null,
source,
animationFrame: null,
videoFrameHandle: null,
@@ -1053,6 +1524,8 @@ export function createColorGradingRuntime(): RuntimeColorGradingApi {
sourceOpacityForCanvas: window.getComputedStyle(element).opacity || "1",
sourceVisibleForCanvas: window.getComputedStyle(element).visibility !== "hidden",
hasDrawn: false,
contextLost: false,
grainSeed: seedForElement(element),
destroyed: false,
};
entries.set(element, entry);
@@ -1158,8 +1631,14 @@ export function createColorGradingRuntime(): RuntimeColorGradingApi {
if (!element) return { state: "missing", message: "Media not found" };
const entry = entries.get(element);
if (entry) {
if (entry.effectError) {
return { state: "unavailable", message: entry.effectError };
}
if (entry.drawError) {
return { state: "unavailable", message: entry.drawError };
}
if (entry.lutError) {
return { state: "unavailable", message: entry.lutError };
return { state: "unavailable", message: `LUT error: ${entry.lutError}` };
}
if (entry.grading.lut && entry.lutLoadingSrc) {
return { state: "pending", message: "Loading LUT" };
packages/engine/src/services/screenshotService.test.ts
+74 -4
View File
@@ -5,11 +5,12 @@ import { type Page } from "puppeteer-core";
import {
pageScreenshotCapture,
cdpSessionCache,
applyDomLayerMask,
removeDomLayerMask,
injectVideoFramesBatch,
syncVideoFrameVisibility,
shouldDefaultCaptureBeyondViewport,
applyDomLayerMask,
removeDomLayerMask,
DOM_LAYER_MASK_STYLE_ID,
} from "./screenshotService.js";
// Stub a Page + CDPSession just enough that pageScreenshotCapture can call
@@ -268,7 +269,7 @@ describe("video-frame injection respects ancestor visibility", () => {
function setupHostHiddenScenario(
hostStyle: StyleLike,
options: { hostAttribute?: HostAttribute } = {},
options: { hostAttribute?: HostAttribute; videoStyle?: StyleLike } = {},
) {
const hostAttribute = options.hostAttribute ?? "data-composition-src";
const hostAttrMarkup =
@@ -310,7 +311,13 @@ describe("video-frame injection respects ancestor visibility", () => {
const styles = new Map<Element, StyleLike>();
styles.set(host, hostStyle);
styles.set(pipFrame, {});
styles.set(video, { opacity: "1", objectFit: "cover", objectPosition: "center", zIndex: "1" });
styles.set(video, {
opacity: "1",
objectFit: "cover",
objectPosition: "center",
zIndex: "1",
...options.videoStyle,
});
Object.defineProperty(window, "getComputedStyle", {
configurable: true,
@@ -498,6 +505,47 @@ describe("video-frame injection respects ancestor visibility", () => {
expect(sibling?.style.visibility).toBe("visible");
});
it("does not copy color-grading source suppression opacity to the injected frame", async () => {
const { teardown, setup } = withGlobals(
setupHostHiddenScenario({}, { videoStyle: { opacity: "0" } }),
);
setup.video.setAttribute("data-hf-color-grading-source-hidden", "true");
try {
await injectVideoFramesBatch(passthroughPage(), [
{
videoId: "pip",
dataUri:
"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkAAIAAAoAAv/lxKUAAAAASUVORK5CYII=",
},
]);
} finally {
teardown();
}
const sibling = setup.video.nextElementSibling as HTMLElement | null;
expect(sibling?.classList.contains("__render_frame__")).toBe(true);
expect(sibling?.style.opacity).toBe("1");
});
it("repairs stale injected-frame opacity while syncing color-graded active videos", async () => {
const { teardown, setup } = withGlobals(setupHostHiddenScenario({}));
setup.video.setAttribute("data-hf-color-grading-source-hidden", "true");
const seededImg = setup.document.createElement("img");
seededImg.classList.add("__render_frame__");
seededImg.style.opacity = "0";
setup.video.parentNode?.insertBefore(seededImg, setup.video.nextSibling);
try {
await syncVideoFrameVisibility(passthroughPage(), ["pip"]);
} finally {
teardown();
}
expect(seededImg.style.opacity).toBe("1");
expect(seededImg.style.visibility).toBe("visible");
});
it("syncVideoFrameVisibility shows the replacement <img> when a plain [data-start] host is visibility:hidden", async () => {
const { teardown, setup } = withGlobals(
setupHostHiddenScenario({ visibility: "hidden" }, { hostAttribute: "data-start" }),
@@ -672,4 +720,26 @@ describe("video-frame injection respects ancestor visibility", () => {
teardown();
}
});
it("applyDomLayerMask carries color grading canvases with their media element", async () => {
const { window, document } = parseHTML(
'<html><head></head><body><div id="root"><video id="pip"></video><canvas id="__hf_color_grading_pip"></canvas></div></body></html>',
);
const teardown = installDomMaskGlobals({ window, document });
try {
await applyDomLayerMask(passthroughPage(), ["pip"], []);
expect(document.getElementById(DOM_LAYER_MASK_STYLE_ID)?.textContent).toContain(
"#__hf_color_grading_pip",
);
await applyDomLayerMask(passthroughPage(), ["root"], ["pip"]);
const canvas = document.getElementById("__hf_color_grading_pip") as HTMLCanvasElement;
expect(canvas.style.visibility).toBe("hidden");
await removeDomLayerMask(passthroughPage(), ["pip"]);
expect(canvas.style.getPropertyValue("visibility") || "").toBe("");
} finally {
teardown();
}
});
});
packages/engine/src/services/screenshotService.ts
+87 -59
View File
@@ -6,9 +6,13 @@
import { type Page } from "puppeteer-core";
import { type CaptureOptions } from "../types.js";
import { MEDIA_VISUAL_STYLE_PROPERTIES } from "@hyperframes/core";
import {
HF_COLOR_GRADING_CANVAS_ID_PREFIX,
MEDIA_VISUAL_STYLE_PROPERTIES,
} from "@hyperframes/core";
export const cdpSessionCache = new WeakMap<Page, import("puppeteer-core").CDPSession>();
const COLOR_GRADING_SOURCE_HIDDEN_ATTR = "data-hf-color-grading-source-hidden";
export async function getCdpSession(page: Page): Promise<import("puppeteer-core").CDPSession> {
let client = cdpSessionCache.get(page);
@@ -279,12 +283,12 @@ const DOM_LAYER_MASK_PREV_PRIORITY_ATTR = "data-hf-dom-layer-mask-prev-priority"
*
* 1. Inject a stylesheet that hides every body descendant
* (`body * { visibility: hidden !important }`) and re-shows the layer's
* elements (and their descendants and their injected `__render_frame_*`
* siblings) via `visibility: visible !important`. CSS `visibility: visible`
* elements (and their descendants, injected `__render_frame_*` siblings,
* and media color-grading canvases) via `visibility: visible !important`. CSS `visibility: visible`
* on a descendant overrides an ancestor's `visibility: hidden`, so deep
* layer elements remain visible even though intermediate parents are
* hidden by the mass-hide rule.
* 2. Inline-hide each `extraHideId` (and its `__render_frame_*` sibling) with
* 2. Inline-hide each `extraHideId` (and its render-frame/color-grading siblings) with
* `visibility: hidden !important`, while first recording its previous
* inline visibility. Inline `!important` beats stylesheet `!important`,
* so this overrides the show rule for elements that fall under a show
@@ -328,6 +332,7 @@ export async function applyDomLayerMask(
hiddenAttr: string;
prevVisibilityAttr: string;
prevPriorityAttr: string;
canvasIdPrefix: string;
}) => {
const existing = document.getElementById(args.styleId);
if (existing) existing.remove();
@@ -390,6 +395,8 @@ export async function applyDomLayerMask(
showSelectors.push(`#${escaped}`, `#${escaped} *`);
const renderEscaped = CSS.escape(`__render_frame_${id}__`);
showSelectors.push(`#${renderEscaped}`, `#${renderEscaped} *`);
const colorGradingEscaped = CSS.escape(`${args.canvasIdPrefix}${id}`);
showSelectors.push(`#${colorGradingEscaped}`, `#${colorGradingEscaped} *`);
}
const massHideRule = "body *{visibility:hidden !important;}";
@@ -416,6 +423,10 @@ export async function applyDomLayerMask(
if (img) {
rememberAndHideElement(img);
}
const colorGradingCanvas = document.getElementById(`${args.canvasIdPrefix}${id}`);
if (colorGradingCanvas instanceof HTMLElement) {
rememberAndHideElement(colorGradingCanvas);
}
}
},
{
@@ -425,6 +436,7 @@ export async function applyDomLayerMask(
hiddenAttr: DOM_LAYER_MASK_HIDDEN_ATTR,
prevVisibilityAttr: DOM_LAYER_MASK_PREV_VISIBILITY_ATTR,
prevPriorityAttr: DOM_LAYER_MASK_PREV_PRIORITY_ATTR,
canvasIdPrefix: HF_COLOR_GRADING_CANVAS_ID_PREFIX,
},
);
}
@@ -434,7 +446,7 @@ export async function applyDomLayerMask(
*
* Removes the mask stylesheet and restores the inline `visibility` values
* temporarily overwritten for hidden timed descendants, `extraHideIds`, and
* their `__render_frame_*` siblings.
* their render-frame/color-grading siblings.
*
* IMPORTANT: We do NOT strip inline `opacity` here. applyDomLayerMask only
* ever sets `visibility` (never `opacity`), so any inline opacity present on
@@ -492,7 +504,11 @@ export async function injectVideoFramesBatch(
): Promise<string[]> {
if (updates.length === 0) return [];
return await page.evaluate(
async (items: Array<{ videoId: string; dataUri: string }>, visualProperties: string[]) => {
async (
items: Array<{ videoId: string; dataUri: string }>,
visualProperties: string[],
colorGradingSourceHiddenAttr: string,
) => {
const injectedIds: string[] = [];
const pendingDecodes: Array<Promise<void>> = [];
const replacementLayoutProperties = new Set([
@@ -570,7 +586,11 @@ export async function injectVideoFramesBatch(
// `opacity: 0`), so its computed opacity is preserved across seeks
// and accurately reflects the user's intent on every frame.
const opacityParsed = parseFloat(computedStyle.opacity);
const computedOpacity = Number.isNaN(opacityParsed) ? 1 : opacityParsed;
const computedOpacity = video.hasAttribute(colorGradingSourceHiddenAttr)
? 1
: Number.isNaN(opacityParsed)
? 1
: opacityParsed;
if (isNewImage) {
img = document.createElement("img");
@@ -655,6 +675,7 @@ export async function injectVideoFramesBatch(
},
updates,
[...MEDIA_VISUAL_STYLE_PROPERTIES],
COLOR_GRADING_SOURCE_HIDDEN_ATTR,
);
}
@@ -662,59 +683,66 @@ export async function syncVideoFrameVisibility(
page: Page,
activeVideoIds: string[],
): Promise<void> {
await page.evaluate((ids: string[]) => {
// Mirror the ancestor-visibility guard from `injectVideoFramesBatch`.
// See that copy for the full rationale on why `visibility: hidden` is
// narrowed to sub-composition hosts only — keep these two functions in
// sync so the inactive-arm decision matches the inject-time decision.
const isVisualAncestorHidden = (el: HTMLElement): boolean => {
let parent = el.parentElement;
while (parent !== null && parent !== document.documentElement) {
const computed = window.getComputedStyle(parent);
if (computed.display === "none") return true;
if (
computed.visibility === "hidden" &&
(parent.hasAttribute("data-composition-src") ||
parent.hasAttribute("data-composition-file"))
) {
return true;
await page.evaluate(
(ids: string[], colorGradingSourceHiddenAttr: string) => {
// Mirror the ancestor-visibility guard from `injectVideoFramesBatch`.
// See that copy for the full rationale on why `visibility: hidden` is
// narrowed to sub-composition hosts only — keep these two functions in
// sync so the inactive-arm decision matches the inject-time decision.
const isVisualAncestorHidden = (el: HTMLElement): boolean => {
let parent = el.parentElement;
while (parent !== null && parent !== document.documentElement) {
const computed = window.getComputedStyle(parent);
if (computed.display === "none") return true;
if (
computed.visibility === "hidden" &&
(parent.hasAttribute("data-composition-src") ||
parent.hasAttribute("data-composition-file"))
) {
return true;
}
parent = parent.parentElement;
}
parent = parent.parentElement;
}
return false;
};
const active = new Set(ids);
const videos = Array.from(document.querySelectorAll("video[data-start]")) as HTMLVideoElement[];
for (const video of videos) {
const img = video.nextElementSibling as HTMLElement | null;
const hasImg = img && img.classList.contains("__render_frame__");
const ancestorHidden = isVisualAncestorHidden(video);
if (active.has(video.id) && !ancestorHidden) {
// Active video: show injected <img>, hide native <video>.
// Do NOT clobber inline opacity here — GSAP-controlled opacity must
// survive until injectVideoFramesBatch reads it via getComputedStyle.
// visibility:hidden alone hides the native element without affecting
// its computed opacity.
video.style.setProperty("visibility", "hidden", "important");
video.style.setProperty("pointer-events", "none", "important");
if (hasImg) {
img.style.visibility = "visible";
}
} else {
// Inactive (or ancestor-hidden) video: hide both. Use visibility only
// (never opacity) so we never clobber GSAP-controlled inline opacity.
// Use `!important` on the <img> hide so `applyDomLayerMask`'s
// important stylesheet rule (`#${showId} *{visibility:visible !important}`)
// cannot revive a stale frame when the sub-comp host lands in the
// active layer's `show` set — same mask-defense reasoning as the
// `isVisualAncestorHidden` branch in `injectVideoFramesBatch`.
video.style.removeProperty("display");
video.style.setProperty("visibility", "hidden", "important");
video.style.setProperty("pointer-events", "none", "important");
if (hasImg) {
img.style.setProperty("visibility", "hidden", "important");
return false;
};
const active = new Set(ids);
const videos = Array.from(
document.querySelectorAll("video[data-start]"),
) as HTMLVideoElement[];
for (const video of videos) {
const img = video.nextElementSibling as HTMLElement | null;
const hasImg = img && img.classList.contains("__render_frame__");
const ancestorHidden = isVisualAncestorHidden(video);
if (active.has(video.id) && !ancestorHidden) {
// Active video: show injected <img>, hide native <video>.
// Do NOT clobber inline opacity here — GSAP-controlled opacity must
// survive until injectVideoFramesBatch reads it via getComputedStyle.
// visibility:hidden alone hides the native element without affecting
// its computed opacity.
video.style.setProperty("visibility", "hidden", "important");
video.style.setProperty("pointer-events", "none", "important");
if (hasImg) {
if (video.hasAttribute(colorGradingSourceHiddenAttr)) img.style.opacity = "1";
img.style.visibility = "visible";
}
} else {
// Inactive (or ancestor-hidden) video: hide both. Use visibility only
// (never opacity) so we never clobber GSAP-controlled inline opacity.
// Use `!important` on the <img> hide so `applyDomLayerMask`'s
// important stylesheet rule (`#${showId} *{visibility:visible !important}`)
// cannot revive a stale frame when the sub-comp host lands in the
// active layer's `show` set — same mask-defense reasoning as the
// `isVisualAncestorHidden` branch in `injectVideoFramesBatch`.
video.style.removeProperty("display");
video.style.setProperty("visibility", "hidden", "important");
video.style.setProperty("pointer-events", "none", "important");
if (hasImg) {
img.style.setProperty("visibility", "hidden", "important");
}
}
}
}
}, activeVideoIds);
},
activeVideoIds,
COLOR_GRADING_SOURCE_HIDDEN_ATTR,
);
}
packages/engine/src/services/videoFrameInjector.ts
+33 -18
View File
@@ -13,6 +13,7 @@ import { type FrameLookupTable } from "./videoFrameExtractor.js";
import { injectVideoFramesBatch, syncVideoFrameVisibility } from "./screenshotService.js";
import { type BeforeCaptureHook } from "./frameCapture.js";
import { DEFAULT_CONFIG, type EngineConfig } from "../config.js";
import { HF_COLOR_GRADING_CANVAS_ID_PREFIX } from "@hyperframes/core";
export interface VideoFrameInjectorOptions extends Partial<
Pick<EngineConfig, "frameDataUriCacheLimit" | "frameDataUriCacheBytesLimitMb">
@@ -275,16 +276,24 @@ export interface VideoElementBounds {
*/
export async function hideVideoElements(page: Page, videoIds: string[]): Promise<void> {
if (videoIds.length === 0) return;
await page.evaluate((ids: string[]) => {
for (const id of ids) {
const el = document.getElementById(id) as HTMLVideoElement | null;
if (el) {
el.style.setProperty("visibility", "hidden", "important");
const img = document.getElementById(`__render_frame_${id}__`);
if (img) img.style.setProperty("visibility", "hidden", "important");
await page.evaluate(
(ids: string[], canvasIdPrefix: string) => {
for (const id of ids) {
const el = document.getElementById(id) as HTMLVideoElement | null;
if (el) {
el.style.setProperty("visibility", "hidden", "important");
const img = document.getElementById(`__render_frame_${id}__`);
if (img) img.style.setProperty("visibility", "hidden", "important");
const colorGradingCanvas = document.getElementById(`${canvasIdPrefix}${id}`);
if (colorGradingCanvas) {
colorGradingCanvas.style.setProperty("visibility", "hidden", "important");
}
}
}
}
}, videoIds);
},
videoIds,
HF_COLOR_GRADING_CANVAS_ID_PREFIX,
);
}
/**
@@ -292,16 +301,22 @@ export async function hideVideoElements(page: Page, videoIds: string[]): Promise
*/
export async function showVideoElements(page: Page, videoIds: string[]): Promise<void> {
if (videoIds.length === 0) return;
await page.evaluate((ids: string[]) => {
for (const id of ids) {
const el = document.getElementById(id) as HTMLVideoElement | null;
if (el) {
el.style.removeProperty("visibility");
const img = document.getElementById(`__render_frame_${id}__`);
if (img) img.style.removeProperty("visibility");
await page.evaluate(
(ids: string[], canvasIdPrefix: string) => {
for (const id of ids) {
const el = document.getElementById(id) as HTMLVideoElement | null;
if (el) {
el.style.removeProperty("visibility");
const img = document.getElementById(`__render_frame_${id}__`);
if (img) img.style.removeProperty("visibility");
const colorGradingCanvas = document.getElementById(`${canvasIdPrefix}${id}`);
if (colorGradingCanvas) colorGradingCanvas.style.removeProperty("visibility");
}
}
}
}, videoIds);
},
videoIds,
HF_COLOR_GRADING_CANVAS_ID_PREFIX,
);
}
/**