fix(core): a graded clip's canvas no longer comes out blank on a real GPU

packages/core/src/runtime/colorGrading.test.ts

@@ -31,7 +31,11 @@ LUT_3D_SIZE 2
 `;
 
 function createMockWebGl(
-  options: { failMediaUpload?: boolean; halfFloatSupported?: boolean } = {},
+  options: {
+    failMediaUpload?: boolean;
+    halfFloatSupported?: boolean;
+    drawingBuffer?: { width: number; height: number };
+  } = {},
 ): WebGLRenderingContext {
   const shader = {};
   const program = {};
@@ -124,6 +128,8 @@ function createMockWebGl(
     drawArrays: vi.fn(),
     deleteTexture: vi.fn(),
     deleteBuffer: vi.fn(),
+    drawingBufferWidth: options.drawingBuffer?.width,
+    drawingBufferHeight: options.drawingBuffer?.height,
   } as unknown as WebGLRenderingContext;
 }
 
@@ -944,6 +950,49 @@ describe("createColorGradingRuntime", () => {
     expect(video.style.getPropertyPriority("opacity")).toBe("important");
   });
 
+  it("caps a clip shown far larger than the screen to what a GPU can draw, keeping its shape", () => {
+    vi.stubGlobal("devicePixelRatio", 1);
+    const video = makeDrawableVideo();
+    Object.defineProperty(video, "offsetWidth", { value: 6582, configurable: true });
+    Object.defineProperty(video, "offsetHeight", { value: 11520, configurable: true });
+
+    const { canvas } = startRuntimeWithVideo(video);
+
+    expect(canvas.width * canvas.height).toBeLessThanOrEqual(1 << 24);
+    expect(canvas.width / canvas.height).toBeCloseTo(6582 / 11520, 2);
+  });
+
+  it("leaves the source showing when the drawing buffer cannot hold the canvas", () => {
+    getContextSpy.mockImplementation((type: string) =>
+      type === "webgl"
+        ? createMockWebGl({ drawingBuffer: { width: 100, height: 100 } })
+        : type === "2d"
+          ? mock2d()
+          : null,
+    );
+
+    const { video } = startRuntimeWithVideo();
+
+    expect(video.style.getPropertyValue("opacity")).not.toBe("0");
+  });
+
+  it("draws a paused video again once its frame is presented, after the first draw sampled nothing", () => {
+    const presented: Array<() => void> = [];
+    const video = makeDrawableVideo();
+    Object.defineProperty(video, "paused", { value: true, configurable: true });
+    Object.defineProperty(video, "requestVideoFrameCallback", {
+      value: (callback: () => void) => presented.push(callback),
+      configurable: true,
+    });
+
+    startRuntimeWithVideo(video);
+    const drawsBefore = texImage2DCalls.length;
+    presented.shift()?.();
+
+    expect(texImage2DCalls.length).toBeGreaterThan(drawsBefore);
+    expect(presented).toHaveLength(0);
+  });
+
   it("keeps the canvas visible when producer render-frame injection hides the source video", () => {
     const video = makeDrawableVideo();
     Object.defineProperty(video, "readyState", {

packages/core/src/runtime/colorGrading.ts

@@ -320,6 +320,9 @@ type LutCacheEntry =
 const LUT_CACHE = new Map<string, LutCacheEntry>();
 const COLOR_GRADING_CANVAS_ATTR = "data-hf-color-grading-canvas";
 const COLOR_GRADING_CANVAS_CLASS = "__hf_color_grading_canvas__";
+// A WebGL drawing buffer past these limits is silently clamped by the GPU, and what it draws is blank.
+const MAX_GRADING_CANVAS_PIXELS = 1 << 24;
+const MAX_GRADING_CANVAS_SIDE = 8192;
 
 /** Captures authored media opacity before animation or grading can mutate it. */
 export function installAuthoredOpacityCapture(): void {
@@ -2900,6 +2903,18 @@ function resolvedLayoutSize(primary: number, fallback: number): number {
   return Math.max(0, Math.round(primary > 0 ? primary : fallback));
 }
 
+/** The pixel size to draw a clip at: its own, or smaller in the same shape when the GPU could not draw it, as for a
+ * clip zoomed far past the screen. */
+function fitToGpuLimits(width: number, height: number): { width: number; height: number } {
+  const scale = Math.min(
+    1,
+    Math.sqrt(MAX_GRADING_CANVAS_PIXELS / (width * height)),
+    MAX_GRADING_CANVAS_SIDE / Math.max(width, height),
+  );
+  const fitted = { width: Math.floor(width * scale), height: Math.floor(height * scale) };
+  return scale < 1 ? fitted : { width: Math.round(width), height: Math.round(height) };
+}
+
 function updateCanvasLayout(
   entry: ColorGradingEntry,
   styleSource: HTMLElement,
@@ -2932,8 +2947,7 @@ function updateCanvasLayout(
     return null;
   }
   const pixelRatio = Math.max(1, window.devicePixelRatio || 1);
-  const width = Math.round(cssWidth * pixelRatio);
-  const height = Math.round(cssHeight * pixelRatio);
+  const { width, height } = fitToGpuLimits(cssWidth * pixelRatio, cssHeight * pixelRatio);
   if (canvas.width !== width) canvas.width = width;
   if (canvas.height !== height) canvas.height = height;
   return { width, height };
@@ -3497,6 +3511,9 @@ function renderGradedFrame(
     frameTime,
   );
   drawFullscreenQuad(gl, { quad: program.quad, position: shader.position });
+  if (gl.drawingBufferWidth < layout.width || gl.drawingBufferHeight < layout.height) {
+    throw new Error("The GPU's drawing buffer is smaller than the grading canvas");
+  }
   presentEntry(entry, layout);
 }
 
@@ -3772,10 +3789,16 @@ function installEntryListeners(entry: ColorGradingEntry): void {
   const redraw = () => {
     drawEntry(entry);
   };
+  // A video says it has data before the GPU can upload it, so this draw can sample nothing; the draw once the
+  // frame is presented is the one that holds the picture.
+  const redrawOnPresentedFrame = () => {
+    drawEntry(entry);
+    scheduleVideoDraw(entry);
+  };
   addListener(entry, entry.element, "load", redraw);
   addListener(entry, entry.element, "loadedmetadata", redraw);
-  addListener(entry, entry.element, "loadeddata", redraw);
-  addListener(entry, entry.element, "seeked", redraw);
+  addListener(entry, entry.element, "loadeddata", redrawOnPresentedFrame);
+  addListener(entry, entry.element, "seeked", redrawOnPresentedFrame);
   addListener(entry, entry.element, "timeupdate", redraw);
   addListener(entry, window, "resize", redraw);
   if (isVideoElement(entry.element)) {
@@ -4007,7 +4030,7 @@ export function createColorGradingRuntime(pausedMediaLease?: {
     trackedElements.add(element);
     installEntryListeners(entry);
     drawEntry(entry);
-    if (isVideoElement(element) && !element.paused) scheduleVideoDraw(entry);
+    if (isVideoElement(element)) scheduleVideoDraw(entry);
     return true;
   };