mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-02 19:57:13 +00:00
perf(core): every graded clip draws through one WebGL context, not one per clip (#5809)
This commit is contained in:
@@ -122,6 +122,11 @@ function createMockWebGl(
|
||||
} as unknown as WebGLRenderingContext;
|
||||
}
|
||||
|
||||
// Each clip shows its pixels on a 2D canvas, copied from the one shared WebGL canvas.
|
||||
function mock2d(): CanvasRenderingContext2D {
|
||||
return { clearRect: vi.fn(), drawImage: vi.fn() } as unknown as CanvasRenderingContext2D;
|
||||
}
|
||||
|
||||
function makeDrawableVideo(): HTMLVideoElement {
|
||||
const video = document.createElement("video");
|
||||
video.id = "hero-video";
|
||||
@@ -182,6 +187,8 @@ describe("createColorGradingRuntime", () => {
|
||||
let getContextSpy: ReturnType<typeof vi.spyOn>;
|
||||
let runtime: RuntimeColorGradingApi | null = null;
|
||||
|
||||
const webglContexts = () => getContextSpy.mock.calls.filter(([type]) => type === "webgl").length;
|
||||
|
||||
beforeEach(() => {
|
||||
document.body.innerHTML = "";
|
||||
lastUniform1f = null;
|
||||
@@ -192,7 +199,7 @@ describe("createColorGradingRuntime", () => {
|
||||
getContextSpy = vi
|
||||
.spyOn(HTMLCanvasElement.prototype, "getContext")
|
||||
.mockImplementation((type: string) =>
|
||||
type === "webgl" ? createMockWebGl() : null,
|
||||
type === "webgl" ? createMockWebGl() : type === "2d" ? mock2d() : null,
|
||||
) as ReturnType<typeof vi.spyOn>;
|
||||
});
|
||||
|
||||
@@ -355,11 +362,41 @@ describe("createColorGradingRuntime", () => {
|
||||
const recreated = document.querySelector<HTMLCanvasElement>("[data-hf-color-grading-canvas]");
|
||||
expect(recreated).not.toBeNull();
|
||||
expect(recreated).toBe(canvas);
|
||||
expect(getContextSpy).toHaveBeenCalledTimes(1);
|
||||
expect(webglContexts()).toBe(1);
|
||||
expect(video.style.getPropertyValue("opacity")).toBe("0");
|
||||
expect(video.style.getPropertyPriority("opacity")).toBe("important");
|
||||
});
|
||||
|
||||
it("draws every graded clip through one WebGL context, however often a seek run shows and hides them", () => {
|
||||
const clips = Array.from({ length: 6 }, (_, i) => {
|
||||
const video = makeDrawableVideo();
|
||||
video.id = `clip-${i}`;
|
||||
video.style.display = "none";
|
||||
document.body.appendChild(video);
|
||||
return video;
|
||||
});
|
||||
runtime = createColorGradingRuntime();
|
||||
|
||||
// Each seek shows two neighbouring clips and hides the rest, as scenes come and go.
|
||||
for (const at of [0, 2, 4, 1, 3, 5, 0, 4]) {
|
||||
clips.forEach((video, i) => {
|
||||
const shown = i === at || i === (at + 1) % clips.length;
|
||||
video.style.display = shown ? "block" : "none";
|
||||
runtime?.setSourceVisibility(video, shown);
|
||||
});
|
||||
}
|
||||
|
||||
expect(webglContexts()).toBe(1);
|
||||
const shared = getContextSpy.mock.instances[0];
|
||||
const copies = getContextSpy.mock.results
|
||||
.filter((_, i) => getContextSpy.mock.calls[i]?.[0] === "2d")
|
||||
.flatMap(
|
||||
({ value }) => (value as { drawImage: ReturnType<typeof vi.fn> }).drawImage.mock.calls,
|
||||
);
|
||||
expect(copies.length).toBeGreaterThan(0);
|
||||
expect(copies.every(([source]) => source === shared)).toBe(true);
|
||||
});
|
||||
|
||||
it("defers WebGL setup for hidden attributed media until it becomes visible", () => {
|
||||
const video = makeDrawableVideo();
|
||||
video.style.display = "none";
|
||||
@@ -367,7 +404,7 @@ describe("createColorGradingRuntime", () => {
|
||||
|
||||
runtime = createColorGradingRuntime();
|
||||
|
||||
expect(getContextSpy).not.toHaveBeenCalled();
|
||||
expect(webglContexts()).toBe(0);
|
||||
expect(document.querySelector("[data-hf-color-grading-canvas]")).toBeNull();
|
||||
expect(runtime.getStatus(video)).toEqual({
|
||||
state: "pending",
|
||||
@@ -376,7 +413,7 @@ describe("createColorGradingRuntime", () => {
|
||||
|
||||
video.style.display = "block";
|
||||
expect(runtime.setSourceVisibility(video, true)).toBe(true);
|
||||
expect(getContextSpy).toHaveBeenCalledTimes(1);
|
||||
expect(webglContexts()).toBe(1);
|
||||
expect(document.querySelector("[data-hf-color-grading-canvas]")).not.toBeNull();
|
||||
});
|
||||
|
||||
@@ -406,12 +443,12 @@ describe("createColorGradingRuntime", () => {
|
||||
{ id: "warm", dataUrl: "data:image/png;base64,preview" },
|
||||
],
|
||||
});
|
||||
expect(getContextSpy).toHaveBeenCalledTimes(1);
|
||||
expect(webglContexts()).toBe(1);
|
||||
expect(document.querySelector("[data-hf-color-grading-canvas]")).toBeNull();
|
||||
expect(video.style.opacity).toBe("");
|
||||
|
||||
await runtime.renderPreviews("#hero-video", [{ id: "mono", grading: "mono-clean" }]);
|
||||
expect(getContextSpy).toHaveBeenCalledTimes(1);
|
||||
expect(webglContexts()).toBe(1);
|
||||
|
||||
const retinaBatch = await runtime.renderPreviews(
|
||||
"#hero-video",
|
||||
@@ -717,7 +754,7 @@ describe("createColorGradingRuntime", () => {
|
||||
|
||||
runtime = createColorGradingRuntime();
|
||||
|
||||
expect(getContextSpy).toHaveBeenCalledTimes(1);
|
||||
expect(webglContexts()).toBe(1);
|
||||
if (!lastUniform1f) throw new Error("Expected WebGL uniform calls");
|
||||
expect(lastUniform1f).toHaveBeenCalledWith("u_intensity", 0);
|
||||
expect(lastUniform1f).toHaveBeenCalledWith("u_kuwahara", 0);
|
||||
@@ -1037,7 +1074,7 @@ describe("createColorGradingRuntime", () => {
|
||||
|
||||
runtime = createColorGradingRuntime();
|
||||
|
||||
expect(getContextSpy).toHaveBeenCalledTimes(1);
|
||||
expect(webglContexts()).toBe(1);
|
||||
expect(document.querySelector("[data-hf-color-grading-canvas]")).not.toBeNull();
|
||||
});
|
||||
|
||||
@@ -1079,7 +1116,7 @@ describe("createColorGradingRuntime", () => {
|
||||
expect(runtime.setSourceVisibility(video, false)).toBe(true);
|
||||
runtime.refresh();
|
||||
expect(canvas.isConnected).toBe(false);
|
||||
expect(getContextSpy).toHaveBeenCalledTimes(1);
|
||||
expect(webglContexts()).toBe(1);
|
||||
expect(video.hasAttribute("data-hf-color-grading-source-hidden")).toBe(false);
|
||||
});
|
||||
|
||||
@@ -1329,7 +1366,11 @@ describe("createColorGradingRuntime", () => {
|
||||
|
||||
it("falls back to the untreated shader output when half-float targets are unavailable", () => {
|
||||
getContextSpy.mockImplementation((type: string) =>
|
||||
type === "webgl" ? createMockWebGl({ halfFloatSupported: false }) : null,
|
||||
type === "webgl"
|
||||
? createMockWebGl({ halfFloatSupported: false })
|
||||
: type === "2d"
|
||||
? mock2d()
|
||||
: null,
|
||||
);
|
||||
const video = makeDrawableVideo();
|
||||
video.setAttribute(
|
||||
@@ -1785,7 +1826,11 @@ describe("createColorGradingRuntime", () => {
|
||||
|
||||
it("reports media texture upload failures in runtime status", () => {
|
||||
getContextSpy.mockImplementation((type: string) =>
|
||||
type === "webgl" ? createMockWebGl({ failMediaUpload: true }) : null,
|
||||
type === "webgl"
|
||||
? createMockWebGl({ failMediaUpload: true })
|
||||
: type === "2d"
|
||||
? mock2d()
|
||||
: null,
|
||||
);
|
||||
const video = makeDrawableVideo();
|
||||
document.body.appendChild(video);
|
||||
@@ -1800,9 +1845,10 @@ describe("createColorGradingRuntime", () => {
|
||||
|
||||
it("falls back to the source media on WebGL context loss and redraws after restore", () => {
|
||||
const { video, canvas } = startRuntimeWithVideo();
|
||||
const shared = getContextSpy.mock.instances[0] as HTMLCanvasElement;
|
||||
|
||||
const lost = new Event("webglcontextlost", { cancelable: true });
|
||||
canvas.dispatchEvent(lost);
|
||||
shared.dispatchEvent(lost);
|
||||
|
||||
expect(lost.defaultPrevented).toBe(true);
|
||||
expect(canvas.style.display).toBe("none");
|
||||
@@ -1813,7 +1859,7 @@ describe("createColorGradingRuntime", () => {
|
||||
message: "WebGL context lost",
|
||||
});
|
||||
|
||||
canvas.dispatchEvent(new Event("webglcontextrestored"));
|
||||
shared.dispatchEvent(new Event("webglcontextrestored"));
|
||||
|
||||
expect(runtime?.getStatus("#hero-video").state).toBe("active");
|
||||
expect(video.hasAttribute("data-hf-color-grading-source-hidden")).toBe(true);
|
||||
|
||||
@@ -192,6 +192,7 @@ interface EffectRenderState {
|
||||
|
||||
interface ColorGradingRenderer extends EffectRenderState {
|
||||
canvas: HTMLCanvasElement;
|
||||
glCanvas: HTMLCanvasElement;
|
||||
}
|
||||
|
||||
interface ColorGradingEntry extends ColorGradingRenderer {
|
||||
@@ -1657,6 +1658,11 @@ function createProgramInfo(canvas: HTMLCanvasElement): {
|
||||
premultipliedAlpha: false,
|
||||
});
|
||||
if (!gl) return null;
|
||||
const program = createProgramResources(gl);
|
||||
return program ? { gl, program } : null;
|
||||
}
|
||||
|
||||
function createProgramResources(gl: WebGLRenderingContext): ProgramInfo | null {
|
||||
const program = createProgram(gl);
|
||||
const texture = createTexture(gl);
|
||||
const lutTexture = createTexture(gl, gl.NEAREST);
|
||||
@@ -1674,57 +1680,54 @@ function createProgramInfo(canvas: HTMLCanvasElement): {
|
||||
gl.bufferData(gl.ARRAY_BUFFER, new Float32Array([-1, -1, 1, -1, -1, 1, 1, 1]), gl.STATIC_DRAW);
|
||||
|
||||
return {
|
||||
gl,
|
||||
program: {
|
||||
program,
|
||||
texture,
|
||||
lutTexture,
|
||||
advancedTexture,
|
||||
advancedSignature: null,
|
||||
quad,
|
||||
position: gl.getAttribLocation(program, "a_pos"),
|
||||
source: gl.getUniformLocation(program, "u_source"),
|
||||
blurSource: gl.getUniformLocation(program, "u_blurSource"),
|
||||
bloomSource: gl.getUniformLocation(program, "u_bloomSource"),
|
||||
kuwaharaSource: gl.getUniformLocation(program, "u_kuwaharaSource"),
|
||||
lut: gl.getUniformLocation(program, "u_lut"),
|
||||
advanced: gl.getUniformLocation(program, "u_advanced"),
|
||||
resolution: gl.getUniformLocation(program, "u_resolution"),
|
||||
uvScale: gl.getUniformLocation(program, "u_uvScale"),
|
||||
uvOffset: gl.getUniformLocation(program, "u_uvOffset"),
|
||||
blurReady: gl.getUniformLocation(program, "u_blurReady"),
|
||||
bloomReady: gl.getUniformLocation(program, "u_bloomReady"),
|
||||
kuwaharaReady: gl.getUniformLocation(program, "u_kuwaharaReady"),
|
||||
lutEnabled: gl.getUniformLocation(program, "u_lutEnabled"),
|
||||
lutSize: gl.getUniformLocation(program, "u_lutSize"),
|
||||
lutTextureSize: gl.getUniformLocation(program, "u_lutTextureSize"),
|
||||
lutDomainMin: gl.getUniformLocation(program, "u_lutDomainMin"),
|
||||
lutDomainMax: gl.getUniformLocation(program, "u_lutDomainMax"),
|
||||
lutIntensity: gl.getUniformLocation(program, "u_lutIntensity"),
|
||||
shadowWheel: gl.getUniformLocation(program, "u_shadowWheel"),
|
||||
midtoneWheel: gl.getUniformLocation(program, "u_midtoneWheel"),
|
||||
highlightWheel: gl.getUniformLocation(program, "u_highlightWheel"),
|
||||
rgbCurvesEnabled: gl.getUniformLocation(program, "u_rgbCurvesEnabled"),
|
||||
hueCurvesEnabled: gl.getUniformLocation(program, "u_hueCurvesEnabled"),
|
||||
secondaryCount: gl.getUniformLocation(program, "u_secondaryCount"),
|
||||
adjustUniforms: createFloatUniformBindings(gl, program, HF_COLOR_GRADING_ADJUST_KEYS),
|
||||
detailUniforms: createFloatUniformBindings(gl, program, HF_COLOR_GRADING_DETAIL_KEYS),
|
||||
effectUniforms: createFloatUniformBindings(gl, program, HF_COLOR_GRADING_EFFECT_KEYS),
|
||||
grainSeed: gl.getUniformLocation(program, "u_grainSeed"),
|
||||
effectTime: gl.getUniformLocation(program, "u_effectTime"),
|
||||
paletteSize: gl.getUniformLocation(program, "u_paletteSize"),
|
||||
palette0: gl.getUniformLocation(program, "u_palette0"),
|
||||
palette1: gl.getUniformLocation(program, "u_palette1"),
|
||||
palette2: gl.getUniformLocation(program, "u_palette2"),
|
||||
palette3: gl.getUniformLocation(program, "u_palette3"),
|
||||
palette4: gl.getUniformLocation(program, "u_palette4"),
|
||||
palette5: gl.getUniformLocation(program, "u_palette5"),
|
||||
intensity: gl.getUniformLocation(program, "u_intensity"),
|
||||
compareEnabled: gl.getUniformLocation(program, "u_compareEnabled"),
|
||||
comparePosition: gl.getUniformLocation(program, "u_comparePosition"),
|
||||
compareSoftness: gl.getUniformLocation(program, "u_compareSoftness"),
|
||||
compareLineWidth: gl.getUniformLocation(program, "u_compareLineWidth"),
|
||||
},
|
||||
program,
|
||||
texture,
|
||||
lutTexture,
|
||||
advancedTexture,
|
||||
advancedSignature: null,
|
||||
quad,
|
||||
position: gl.getAttribLocation(program, "a_pos"),
|
||||
source: gl.getUniformLocation(program, "u_source"),
|
||||
blurSource: gl.getUniformLocation(program, "u_blurSource"),
|
||||
bloomSource: gl.getUniformLocation(program, "u_bloomSource"),
|
||||
kuwaharaSource: gl.getUniformLocation(program, "u_kuwaharaSource"),
|
||||
lut: gl.getUniformLocation(program, "u_lut"),
|
||||
advanced: gl.getUniformLocation(program, "u_advanced"),
|
||||
resolution: gl.getUniformLocation(program, "u_resolution"),
|
||||
uvScale: gl.getUniformLocation(program, "u_uvScale"),
|
||||
uvOffset: gl.getUniformLocation(program, "u_uvOffset"),
|
||||
blurReady: gl.getUniformLocation(program, "u_blurReady"),
|
||||
bloomReady: gl.getUniformLocation(program, "u_bloomReady"),
|
||||
kuwaharaReady: gl.getUniformLocation(program, "u_kuwaharaReady"),
|
||||
lutEnabled: gl.getUniformLocation(program, "u_lutEnabled"),
|
||||
lutSize: gl.getUniformLocation(program, "u_lutSize"),
|
||||
lutTextureSize: gl.getUniformLocation(program, "u_lutTextureSize"),
|
||||
lutDomainMin: gl.getUniformLocation(program, "u_lutDomainMin"),
|
||||
lutDomainMax: gl.getUniformLocation(program, "u_lutDomainMax"),
|
||||
lutIntensity: gl.getUniformLocation(program, "u_lutIntensity"),
|
||||
shadowWheel: gl.getUniformLocation(program, "u_shadowWheel"),
|
||||
midtoneWheel: gl.getUniformLocation(program, "u_midtoneWheel"),
|
||||
highlightWheel: gl.getUniformLocation(program, "u_highlightWheel"),
|
||||
rgbCurvesEnabled: gl.getUniformLocation(program, "u_rgbCurvesEnabled"),
|
||||
hueCurvesEnabled: gl.getUniformLocation(program, "u_hueCurvesEnabled"),
|
||||
secondaryCount: gl.getUniformLocation(program, "u_secondaryCount"),
|
||||
adjustUniforms: createFloatUniformBindings(gl, program, HF_COLOR_GRADING_ADJUST_KEYS),
|
||||
detailUniforms: createFloatUniformBindings(gl, program, HF_COLOR_GRADING_DETAIL_KEYS),
|
||||
effectUniforms: createFloatUniformBindings(gl, program, HF_COLOR_GRADING_EFFECT_KEYS),
|
||||
grainSeed: gl.getUniformLocation(program, "u_grainSeed"),
|
||||
effectTime: gl.getUniformLocation(program, "u_effectTime"),
|
||||
paletteSize: gl.getUniformLocation(program, "u_paletteSize"),
|
||||
palette0: gl.getUniformLocation(program, "u_palette0"),
|
||||
palette1: gl.getUniformLocation(program, "u_palette1"),
|
||||
palette2: gl.getUniformLocation(program, "u_palette2"),
|
||||
palette3: gl.getUniformLocation(program, "u_palette3"),
|
||||
palette4: gl.getUniformLocation(program, "u_palette4"),
|
||||
palette5: gl.getUniformLocation(program, "u_palette5"),
|
||||
intensity: gl.getUniformLocation(program, "u_intensity"),
|
||||
compareEnabled: gl.getUniformLocation(program, "u_compareEnabled"),
|
||||
comparePosition: gl.getUniformLocation(program, "u_comparePosition"),
|
||||
compareSoftness: gl.getUniformLocation(program, "u_compareSoftness"),
|
||||
compareLineWidth: gl.getUniformLocation(program, "u_compareLineWidth"),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1936,11 +1939,10 @@ function destroyMainProgramResources(gl: WebGLRenderingContext, program: Program
|
||||
}
|
||||
|
||||
function replaceProgramResources(entry: ColorGradingEntry): boolean {
|
||||
const created = createProgramInfo(entry.canvas);
|
||||
if (!created) return false;
|
||||
const program = createProgramResources(entry.gl);
|
||||
if (!program) return false;
|
||||
destroyProgramResources(entry);
|
||||
entry.gl = created.gl;
|
||||
entry.program = created.program;
|
||||
entry.program = program;
|
||||
entry.lut = null;
|
||||
entry.lutLoadingSrc = null;
|
||||
entry.lutError = null;
|
||||
@@ -3100,24 +3102,76 @@ function bindProgramTextures(
|
||||
}
|
||||
|
||||
// fallow-ignore-next-line complexity
|
||||
function drawEntry(entry: ColorGradingEntry): boolean {
|
||||
if (entry.destroyed || entry.contextLost) return false;
|
||||
function fitSharedCanvas(entry: ColorGradingEntry, size: { width: number; height: number }): void {
|
||||
if (entry.glCanvas === entry.canvas) return;
|
||||
if (entry.glCanvas.width < size.width) entry.glCanvas.width = size.width;
|
||||
if (entry.glCanvas.height < size.height) entry.glCanvas.height = size.height;
|
||||
}
|
||||
|
||||
function presentEntry(entry: ColorGradingEntry, size: { width: number; height: number }): void {
|
||||
if (entry.glCanvas === entry.canvas) return;
|
||||
const context = entry.canvas.getContext("2d");
|
||||
if (!context) throw new Error("Canvas 2D is unavailable");
|
||||
const { width, height } = size;
|
||||
context.clearRect(0, 0, width, height);
|
||||
context.drawImage(
|
||||
entry.glCanvas,
|
||||
0,
|
||||
entry.glCanvas.height - height,
|
||||
width,
|
||||
height,
|
||||
0,
|
||||
0,
|
||||
width,
|
||||
height,
|
||||
);
|
||||
}
|
||||
|
||||
type DrawableFrame = {
|
||||
source: TexImageSource;
|
||||
pixels: NonNullable<ReturnType<typeof readablePixels>>;
|
||||
size: { width: number; height: number };
|
||||
};
|
||||
|
||||
function drawableFrame(entry: ColorGradingEntry): DrawableFrame | null {
|
||||
const source = getDrawableSource(entry.element);
|
||||
const pixels = source && readablePixels(source, entry);
|
||||
if (!source || !pixels) {
|
||||
if (!entry.hasDrawn) entry.canvas.style.display = "none";
|
||||
return false;
|
||||
return null;
|
||||
}
|
||||
const sourceSize = readSourceSize(source);
|
||||
if (!sourceSize) return false;
|
||||
const styleSource = isHtmlElement(source) ? source : entry.element;
|
||||
const sourceOpacity = entry.element.style.getPropertyValue("opacity");
|
||||
const sourceOpacityPriority = entry.element.style.getPropertyPriority("opacity");
|
||||
const hiddenByColorGrading =
|
||||
entry.sourceHidden && sourceOpacity === "0" && sourceOpacityPriority === "important";
|
||||
const size = readSourceSize(source);
|
||||
return size ? { source, pixels, size } : null;
|
||||
}
|
||||
|
||||
function restoreAuthoredOpacity(entry: ColorGradingEntry): void {
|
||||
if (entry.sourceInlineOpacity === null) {
|
||||
entry.element.style.removeProperty("opacity");
|
||||
return;
|
||||
}
|
||||
entry.element.style.setProperty(
|
||||
"opacity",
|
||||
entry.sourceInlineOpacity,
|
||||
entry.sourceInlineOpacityPriority || "",
|
||||
);
|
||||
}
|
||||
|
||||
function isHiddenByColorGrading(entry: ColorGradingEntry): boolean {
|
||||
const { style } = entry.element;
|
||||
return (
|
||||
entry.sourceHidden &&
|
||||
style.getPropertyValue("opacity") === "0" &&
|
||||
style.getPropertyPriority("opacity") === "important"
|
||||
);
|
||||
}
|
||||
|
||||
function syncSourceAppearance(
|
||||
entry: ColorGradingEntry,
|
||||
source: TexImageSource,
|
||||
injectedFrameSource: boolean,
|
||||
): void {
|
||||
const hiddenByColorGrading = isHiddenByColorGrading(entry);
|
||||
const sourceVisibility = entry.element.style.getPropertyValue("visibility");
|
||||
const injectedFrameSource = isRenderFrameImage(source);
|
||||
if (injectedFrameSource) keepCanvasAboveSource(entry, source);
|
||||
// When color grading hid the source (opacity:0 !important), temporarily
|
||||
// restore the authored inline opacity so getComputedStyle reflects the CSS
|
||||
// animation's current value instead of our own hide. Without this, a CSS
|
||||
@@ -3125,29 +3179,79 @@ function drawEntry(entry: ColorGradingEntry): boolean {
|
||||
// the entire render because the !important blocks all animation effects on
|
||||
// the property (#3329). The restore–read–rehide is synchronous, so no
|
||||
// repaint occurs between the style writes.
|
||||
if (hiddenByColorGrading && !injectedFrameSource) {
|
||||
if (entry.sourceInlineOpacity !== null) {
|
||||
entry.element.style.setProperty(
|
||||
"opacity",
|
||||
entry.sourceInlineOpacity,
|
||||
entry.sourceInlineOpacityPriority || "",
|
||||
);
|
||||
} else {
|
||||
entry.element.style.removeProperty("opacity");
|
||||
}
|
||||
}
|
||||
const computed = window.getComputedStyle(injectedFrameSource ? source : entry.element);
|
||||
const restoresOpacity = hiddenByColorGrading && !injectedFrameSource;
|
||||
if (restoresOpacity) restoreAuthoredOpacity(entry);
|
||||
const computed = window.getComputedStyle(
|
||||
injectedFrameSource ? (source as HTMLImageElement) : entry.element,
|
||||
);
|
||||
// Grading never writes `visibility`, so the source's computed visibility
|
||||
// always tracks the clip window and has to be re-read every frame.
|
||||
entry.sourceOpacityForCanvas = computed.opacity || "1";
|
||||
if (hiddenByColorGrading && !injectedFrameSource) {
|
||||
entry.element.style.setProperty("opacity", "0", "important");
|
||||
}
|
||||
if (restoresOpacity) entry.element.style.setProperty("opacity", "0", "important");
|
||||
entry.sourceVisibleForCanvas =
|
||||
(injectedFrameSource || sourceVisibility !== "hidden") && computed.visibility !== "hidden";
|
||||
const layout = updateCanvasLayout(entry, styleSource);
|
||||
if (!layout) return false;
|
||||
}
|
||||
|
||||
function canonicalFrameTime(element: ColorGradingMediaElement): number {
|
||||
const runtimeTime = (window as WindowWithColorGrading).__player?.getTime?.();
|
||||
if (typeof runtimeTime === "number" && Number.isFinite(runtimeTime))
|
||||
return Math.max(0, runtimeTime);
|
||||
return isVideoElement(element) ? Math.max(0, element.currentTime) : 0;
|
||||
}
|
||||
|
||||
function renderGradedFrame(
|
||||
entry: ColorGradingEntry,
|
||||
pixels: DrawableFrame["pixels"],
|
||||
layout: { width: number; height: number },
|
||||
uv: ReturnType<typeof calculateObjectFitUv>,
|
||||
): void {
|
||||
const { gl, program } = entry;
|
||||
fitSharedCanvas(entry, layout);
|
||||
const grading = readAnimatedGrading(entry);
|
||||
const lut = ensureEntryLut(entry);
|
||||
// Browser media elements are top-left oriented; WebGL texture coordinates
|
||||
// are bottom-left oriented unless the upload is flipped.
|
||||
uploadSourceTexture(gl, program.texture, pixels);
|
||||
const hasAnimatedKuwahara = readAnimatedValue(entry.element, ANIMATED_KUWAHARA_PROPERTY) !== null;
|
||||
const prepared = prepareEffectTextures(entry, grading, layout, uv, {
|
||||
preserveKuwahara: hasAnimatedKuwahara,
|
||||
});
|
||||
|
||||
gl.bindFramebuffer(gl.FRAMEBUFFER, null);
|
||||
gl.viewport(0, 0, layout.width, layout.height);
|
||||
gl.useProgram(program.program);
|
||||
bindProgramTextures(gl, program, prepared);
|
||||
const frameTime = canonicalFrameTime(entry.element);
|
||||
const grainSeed = entry.grainSeed + Math.floor(frameTime * 60);
|
||||
applyUniforms(
|
||||
gl,
|
||||
program,
|
||||
grading,
|
||||
lut,
|
||||
prepared.blurReady,
|
||||
prepared.bloomReady,
|
||||
prepared.kuwaharaReady,
|
||||
entry.compare,
|
||||
layout,
|
||||
uv,
|
||||
grainSeed,
|
||||
frameTime,
|
||||
);
|
||||
drawFullscreenQuad(gl, program);
|
||||
presentEntry(entry, layout);
|
||||
}
|
||||
|
||||
function placeCanvas(
|
||||
entry: ColorGradingEntry,
|
||||
source: TexImageSource,
|
||||
sourceSize: { width: number; height: number },
|
||||
): {
|
||||
layout: { width: number; height: number };
|
||||
uv: ReturnType<typeof calculateObjectFitUv>;
|
||||
} | null {
|
||||
const styleSource = isHtmlElement(source) ? source : entry.element;
|
||||
const layout = updateCanvasLayout(entry, styleSource);
|
||||
if (!layout) return null;
|
||||
const style = window.getComputedStyle(styleSource);
|
||||
const uv = calculateObjectFitUv(
|
||||
layout.width,
|
||||
@@ -3157,46 +3261,22 @@ function drawEntry(entry: ColorGradingEntry): boolean {
|
||||
style.objectFit,
|
||||
style.objectPosition,
|
||||
);
|
||||
const { gl, program } = entry;
|
||||
try {
|
||||
const grading = readAnimatedGrading(entry);
|
||||
const lut = ensureEntryLut(entry);
|
||||
// Browser media elements are top-left oriented; WebGL texture coordinates
|
||||
// are bottom-left oriented unless the upload is flipped.
|
||||
uploadSourceTexture(gl, program.texture, pixels);
|
||||
const hasAnimatedKuwahara =
|
||||
readAnimatedValue(entry.element, ANIMATED_KUWAHARA_PROPERTY) !== null;
|
||||
const prepared = prepareEffectTextures(entry, grading, layout, uv, {
|
||||
preserveKuwahara: hasAnimatedKuwahara,
|
||||
});
|
||||
return { layout, uv };
|
||||
}
|
||||
|
||||
gl.bindFramebuffer(gl.FRAMEBUFFER, null);
|
||||
gl.viewport(0, 0, layout.width, layout.height);
|
||||
gl.useProgram(program.program);
|
||||
bindProgramTextures(gl, program, prepared);
|
||||
const runtimeTime = (window as WindowWithColorGrading).__player?.getTime?.();
|
||||
const frameTime =
|
||||
typeof runtimeTime === "number" && Number.isFinite(runtimeTime)
|
||||
? Math.max(0, runtimeTime)
|
||||
: isVideoElement(entry.element)
|
||||
? Math.max(0, entry.element.currentTime)
|
||||
: 0;
|
||||
const grainSeed = entry.grainSeed + Math.floor(frameTime * 60);
|
||||
applyUniforms(
|
||||
gl,
|
||||
program,
|
||||
grading,
|
||||
lut,
|
||||
prepared.blurReady,
|
||||
prepared.bloomReady,
|
||||
prepared.kuwaharaReady,
|
||||
entry.compare,
|
||||
layout,
|
||||
uv,
|
||||
grainSeed,
|
||||
frameTime,
|
||||
);
|
||||
drawFullscreenQuad(gl, program);
|
||||
function drawEntry(entry: ColorGradingEntry): boolean {
|
||||
if (entry.destroyed || entry.contextLost) return false;
|
||||
const frame = drawableFrame(entry);
|
||||
if (!frame) return false;
|
||||
const { source, pixels, size: sourceSize } = frame;
|
||||
const injectedFrameSource = isRenderFrameImage(source);
|
||||
if (injectedFrameSource) keepCanvasAboveSource(entry, source);
|
||||
syncSourceAppearance(entry, source, injectedFrameSource);
|
||||
const placed = placeCanvas(entry, source, sourceSize);
|
||||
if (!placed) return false;
|
||||
const { layout, uv } = placed;
|
||||
try {
|
||||
renderGradedFrame(entry, pixels, layout, uv);
|
||||
hideSourceElement(entry);
|
||||
if (injectedFrameSource) hideFrameBorder(entry, source);
|
||||
entry.hasDrawn = true;
|
||||
@@ -3231,6 +3311,7 @@ function createPreviewRenderer(): ColorGradingPreviewRenderer | null {
|
||||
return created
|
||||
? {
|
||||
canvas,
|
||||
glCanvas: canvas,
|
||||
...created,
|
||||
lut: null,
|
||||
effectTargets: null,
|
||||
@@ -3433,24 +3514,6 @@ function installEntryListeners(entry: ColorGradingEntry): void {
|
||||
addListener(entry, entry.element, "play", () => scheduleVideoDraw(entry));
|
||||
addListener(entry, entry.element, "pause", redraw);
|
||||
}
|
||||
addListener(entry, entry.canvas, "webglcontextlost", (event) => {
|
||||
event.preventDefault();
|
||||
entry.contextLost = true;
|
||||
entry.drawError = "WebGL context lost";
|
||||
entry.canvas.style.display = "none";
|
||||
restoreSourceElement(entry);
|
||||
});
|
||||
addListener(entry, entry.canvas, "webglcontextrestored", () => {
|
||||
entry.contextLost = false;
|
||||
if (!replaceProgramResources(entry)) {
|
||||
entry.contextLost = true;
|
||||
entry.drawError = "WebGL context restore failed";
|
||||
restoreSourceElement(entry);
|
||||
return;
|
||||
}
|
||||
entry.drawError = null;
|
||||
drawEntry(entry);
|
||||
});
|
||||
if (typeof ResizeObserver !== "undefined") {
|
||||
entry.resizeObserver = new ResizeObserver(redraw);
|
||||
entry.resizeObserver.observe(entry.element);
|
||||
@@ -3503,6 +3566,7 @@ function detachEntry(entry: ColorGradingEntry): ColorGradingRenderer | null {
|
||||
}
|
||||
return {
|
||||
canvas: entry.canvas,
|
||||
glCanvas: entry.glCanvas,
|
||||
gl: entry.gl,
|
||||
program: entry.program,
|
||||
effectTargets: null,
|
||||
@@ -3548,6 +3612,51 @@ export function createColorGradingRuntime(pausedMediaLease?: {
|
||||
let previewRenderer: ColorGradingPreviewRenderer | null = null;
|
||||
let previewQueue = Promise.resolve();
|
||||
let destroyed = false;
|
||||
let shared: { canvas: HTMLCanvasElement; gl: WebGLRenderingContext } | null = null;
|
||||
const trackedEntries = () =>
|
||||
[...trackedElements].flatMap((element) => entries.get(element) ?? []);
|
||||
|
||||
const sharedContext = () => {
|
||||
if (shared) return shared;
|
||||
const canvas = document.createElement("canvas");
|
||||
const gl = canvas.getContext("webgl", { alpha: true, premultipliedAlpha: false });
|
||||
if (!gl) return null;
|
||||
canvas.addEventListener("webglcontextlost", (event) => {
|
||||
event.preventDefault();
|
||||
idleRenderers.length = 0;
|
||||
for (const entry of trackedEntries()) {
|
||||
entry.contextLost = true;
|
||||
entry.drawError = "WebGL context lost";
|
||||
entry.canvas.style.display = "none";
|
||||
restoreSourceElement(entry);
|
||||
}
|
||||
});
|
||||
canvas.addEventListener("webglcontextrestored", () => {
|
||||
for (const entry of trackedEntries()) {
|
||||
entry.contextLost = !replaceProgramResources(entry);
|
||||
entry.drawError = entry.contextLost ? "WebGL context restore failed" : null;
|
||||
if (entry.contextLost) restoreSourceElement(entry);
|
||||
else drawEntry(entry);
|
||||
}
|
||||
});
|
||||
shared = { canvas, gl };
|
||||
return shared;
|
||||
};
|
||||
|
||||
const newRenderer = (element: ColorGradingMediaElement): ColorGradingRenderer | null => {
|
||||
const context = sharedContext();
|
||||
const program = context && createProgramResources(context.gl);
|
||||
if (!context || !program) return null;
|
||||
return {
|
||||
canvas: makeCanvas(element),
|
||||
glCanvas: context.canvas,
|
||||
gl: context.gl,
|
||||
program,
|
||||
effectTargets: null,
|
||||
kuwaharaTargets: null,
|
||||
effectError: null,
|
||||
};
|
||||
};
|
||||
|
||||
const upsert = (
|
||||
element: ColorGradingMediaElement,
|
||||
@@ -3566,20 +3675,9 @@ export function createColorGradingRuntime(pausedMediaLease?: {
|
||||
if (renderer) {
|
||||
attachCanvas(renderer.canvas, element);
|
||||
} else {
|
||||
const canvas = makeCanvas(element);
|
||||
const created = createProgramInfo(canvas);
|
||||
if (!created) {
|
||||
canvas.remove();
|
||||
return false;
|
||||
}
|
||||
renderer = {
|
||||
canvas,
|
||||
gl: created.gl,
|
||||
program: created.program,
|
||||
effectTargets: null,
|
||||
kuwaharaTargets: null,
|
||||
effectError: null,
|
||||
};
|
||||
const created = newRenderer(element);
|
||||
if (!created) return false;
|
||||
renderer = created;
|
||||
}
|
||||
const entry: ColorGradingEntry = {
|
||||
element,
|
||||
@@ -3649,7 +3747,7 @@ export function createColorGradingRuntime(pausedMediaLease?: {
|
||||
const renderer = detachEntry(entry);
|
||||
if (renderer) {
|
||||
if (entry.contextLost || idleRenderers.length >= MAX_IDLE_COLOR_GRADING_RENDERERS) {
|
||||
destroyProgramResources(renderer, true);
|
||||
destroyProgramResources(renderer);
|
||||
} else {
|
||||
idleRenderers.push(renderer);
|
||||
}
|
||||
@@ -3864,8 +3962,10 @@ export function createColorGradingRuntime(pausedMediaLease?: {
|
||||
observer?.disconnect();
|
||||
observer = null;
|
||||
for (const element of trackedElements) removeElement(element);
|
||||
for (const renderer of idleRenderers) destroyProgramResources(renderer, true);
|
||||
for (const renderer of idleRenderers) destroyProgramResources(renderer);
|
||||
idleRenderers.length = 0;
|
||||
shared?.gl.getExtension("WEBGL_lose_context")?.loseContext();
|
||||
shared = null;
|
||||
if (previewRenderer) {
|
||||
destroyProgramResources(previewRenderer, true);
|
||||
previewRenderer = null;
|
||||
|
||||
Reference in New Issue
Block a user