mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-09-07 18:26:17 +00:00
James RussoandClaude Opus 4.8
d580f2a1d8
* fix(render): make WebGL video textures deterministic in headless render WebGL compositions that sample a `<video>` as a texture (e.g. a faceted crystal with clips mapped onto its facets) rendered with flickering, non-deterministic facets: a video would intermittently show a stale frame or go black, and the same frame differed between two renders. Two gaps caused this: 1. No WebGL analog of the WebGPU `patchVideoTextureCompat`. Chrome's headless compositor can't feed decoded `<video>` frames to the GPU, so the engine injects a decoded `<img class="__render_frame__">` sibling per video each frame. The WebGPU `copyExternalImageToTexture` path substitutes it, but `texImage2D` / `texSubImage2D` did not — so WebGL uploaded a stale/black frame. Add `patchWebGLVideoTextureCompat()` mirroring the WebGPU patch (shared `resolveRenderFrameImage` helper). 2. Capture ordering. Per frame the runtime seeks (GPU adapters render on `hf-seek`) BEFORE the engine injects the decoded frames, so the GPU render read a frame that didn't exist yet. After injecting, the engine now calls `window.__hfReseekGpu(t)` — a force-dispatch (`forceDispatchSeekEvent`) that bypasses the same-time `hf-seek` dedup — so GPU compositions re-upload their textures from the freshly-injected, decoded frames, deterministically. Tests: unit tests for the texImage2D/texSubImage2D substitution and the force-dispatch, plus a videoFrameInjector regression test asserting the post-injection GPU reseek fires only when frames were injected. Verified end-to-end: a WebGL prism with 8 live <video> facets renders byte-identical across independent runs with no facet flicker. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> * test(render): add producer render-compat regression for WebGL video textures A WebGL2 canvas samples a <video> as a texture every hf-seek (the natural author pattern, distilled from the HeyGen prism). The render-compat harness renders it and compares against the golden: with the video-texture fix the render reproduces the decoded frames; revert the fix and the canvas renders black, collapsing the comparison. Golden verified to contain real, time-varying video content (not black), so a regression is caught rather than passing vacuously. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
107 lines
4.0 KiB
HTML
107 lines
4.0 KiB
HTML
<!DOCTYPE html>
|
|
<html>
|
|
<head><style data-hyperframes-text-rendering="true">html,body,*{text-rendering:geometricPrecision}</style>
|
|
<style>
|
|
body {
|
|
margin: 0;
|
|
background: #000;
|
|
}
|
|
[data-composition-id="webgl-video-texture-test"] {
|
|
position: relative;
|
|
overflow: hidden;
|
|
}
|
|
#gl {
|
|
position: absolute;
|
|
top: 0;
|
|
left: 0;
|
|
width: 1280px;
|
|
height: 720px;
|
|
}
|
|
/* Texture-source video: sampled into WebGL only, parked off-canvas. */
|
|
#vid {
|
|
position: absolute;
|
|
top: 0;
|
|
left: 0;
|
|
width: 4px;
|
|
height: 4px;
|
|
opacity: 0;
|
|
z-index: -1;
|
|
}
|
|
</style>
|
|
</head>
|
|
<body>
|
|
<div id="root" data-composition-id="webgl-video-texture-test" data-width="1280" data-height="720" data-start="0" data-duration="2">
|
|
<video id="vid" class="clip" data-start="0" data-duration="2" data-track-index="0" src="clip.mp4" muted playsinline data-end="2" data-has-audio="false"></video>
|
|
<canvas id="gl"></canvas>
|
|
|
|
|
|
</div>
|
|
<script>// WebGL2 textured fullscreen triangle sampling the source video element.
|
|
// The handler uploads that video as a texture on every hf-seek — the
|
|
// natural author pattern. In headless render, Chrome can't supply decoded
|
|
// video frames to WebGL, so the core video-texture patch substitutes the
|
|
// engine's injected, decoded render-frame image, and the post-injection
|
|
// GPU reseek re-runs this handler. With the fix, render reproduces the
|
|
// preview snapshot (PSNR high); without it the canvas is black (PSNR
|
|
// collapses). NB: avoid writing literal tag syntax in comments here — the
|
|
// compiler's media-attribute injection scans the raw HTML for it.
|
|
(function () {
|
|
var canvas = document.getElementById("gl");
|
|
canvas.width = 1280;
|
|
canvas.height = 720;
|
|
var gl = canvas.getContext("webgl2", { preserveDrawingBuffer: true });
|
|
if (!gl) return;
|
|
|
|
var VS =
|
|
"#version 300 es\n" +
|
|
"in vec2 p; out vec2 uv;\n" +
|
|
"void main(){ uv = vec2(p.x*0.5+0.5, 0.5-p.y*0.5); gl_Position = vec4(p,0.0,1.0); }";
|
|
var FS =
|
|
"#version 300 es\n" +
|
|
"precision highp float; in vec2 uv; uniform sampler2D tex; out vec4 c;\n" +
|
|
"void main(){ c = texture(tex, uv); }";
|
|
|
|
function sh(type, src) {
|
|
var s = gl.createShader(type);
|
|
gl.shaderSource(s, src);
|
|
gl.compileShader(s);
|
|
return s;
|
|
}
|
|
var prog = gl.createProgram();
|
|
gl.attachShader(prog, sh(gl.VERTEX_SHADER, VS));
|
|
gl.attachShader(prog, sh(gl.FRAGMENT_SHADER, FS));
|
|
gl.linkProgram(prog);
|
|
gl.useProgram(prog);
|
|
|
|
var buf = gl.createBuffer();
|
|
gl.bindBuffer(gl.ARRAY_BUFFER, buf);
|
|
gl.bufferData(gl.ARRAY_BUFFER, new Float32Array([-1, -1, 3, -1, -1, 3]), gl.STATIC_DRAW);
|
|
var loc = gl.getAttribLocation(prog, "p");
|
|
gl.enableVertexAttribArray(loc);
|
|
gl.vertexAttribPointer(loc, 2, gl.FLOAT, false, 0, 0);
|
|
|
|
var tex = gl.createTexture();
|
|
gl.bindTexture(gl.TEXTURE_2D, tex);
|
|
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
|
|
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
|
|
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
|
|
|
|
var video = document.getElementById("vid");
|
|
|
|
function render() {
|
|
if (video && video.readyState >= 2) {
|
|
gl.bindTexture(gl.TEXTURE_2D, tex);
|
|
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, video);
|
|
}
|
|
gl.clearColor(0, 0, 0, 1);
|
|
gl.clear(gl.COLOR_BUFFER_BIT);
|
|
gl.drawArrays(gl.TRIANGLES, 0, 3);
|
|
}
|
|
|
|
window.addEventListener("hf-seek", function () {
|
|
render();
|
|
});
|
|
render();
|
|
})();</script></body>
|
|
</html>
|