mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-09-01 19:42:03 +00:00
feat(core): add TypeGPU/WebGPU runtime adapter (#755)
* feat(core): add TypeGPU/WebGPU runtime adapter
Adds a deterministic seek adapter for compositions that render with
TypeGPU or raw WebGPU. Follows the same push+poll pattern as the
Three.js adapter:
- Sets `window.__hfTypegpuTime` on every seek so render loops can
poll it instead of `performance.now()`.
- Dispatches a `"hf-seek"` CustomEvent on `window` so compositions
can imperatively re-render a single frame at the new seek position.
Compositions listen for the event and update their time uniform:
```js
window.addEventListener("hf-seek", (e) => render(e.detail.time));
```
Works with TypeGPU (docs.swmansion.com/TypeGPU) and raw WebGPU alike.
No assumptions are made about pipeline construction — multiple canvases
or renderers are supported by sharing the same event.
- 9 unit tests, all pass
- wired in init.ts adapter array
- `__hfTypegpuTime` declared in window.d.ts
* fix(core): deduplicate hf-seek dispatch across GPU adapters
Both three and typegpu adapters previously dispatched the same
"hf-seek" CustomEvent independently, causing any composition that
registered a listener to receive two events per seek tick — doubling
per-scrub GPU work even though the renders are idempotent.
Fix: extract a shared `dispatchSeekEvent` helper (seek-dispatch.ts)
that deduplicates by exact float equality within the same synchronous
call stack. Both adapters now call this helper instead of dispatching
directly.
Also adds:
- `resetSeekDispatchState()` export for test isolation
- `beforeEach` reset in three.test.ts and typegpu.test.ts
- New typegpu test: "duplicate seek to same time fires event only once"
- Docstring additions to typegpu.ts: render-mode determinism contract
(await device.queue.onSubmittedWorkDone()) and navigator.gpu feature
detection guidance for composition authors
* feat(core): video-texture render compat + TypeGPU skill
Adds the missing pieces for video-backed WebGPU effects in render mode:
- `video-texture-compat.ts`: monkey-patches `GPUQueue.copyExternalImageToTexture`
to detect the engine's injected `<img class="__render_frame__">` siblings and
transparently substitute them for `<video>` sources. Headless Chrome can't
supply decoded video frames to WebGPU, but the engine's pre-extracted frame
images work. Falls through to the original path in preview mode.
- `patchVideoTextureCompat()` wired in init.ts after adapter array creation.
- `skills/typegpu/SKILL.md`: full authoring guide for TypeGPU/WebGPU compositions
covering contract, timeline registration, video-backed effects, frosted blur
via downsample pass, WGSL patterns, and deterministic rendering.
* test(producer): add typegpu-adapter regression test
Self-contained WebGPU composition with:
- Procedural gradient background (no video dependency)
- Animated ring driven by hf-seek time uniform
- Pulsing center glow
- Two GSAP-driven captions testing adapter sync
Verifies the TypeGPU adapter's hf-seek → WebGPU render pipeline
produces deterministic frames. workers: 1 for consistency.
Note: output.mp4 baseline needs to be generated in CI — the local
Docker image can't launch Chrome (ARM/x86 mismatch on Mac).
This commit is contained in:
@@ -0,0 +1,36 @@
|
||||
import { swallow } from "../diagnostics";
|
||||
|
||||
/**
|
||||
* Shared, deduplicated `"hf-seek"` CustomEvent dispatcher for GPU adapters.
|
||||
*
|
||||
* Both the Three.js and TypeGPU adapters dispatch the same `"hf-seek"` event
|
||||
* so that compositions need not know which GPU library they're paired with.
|
||||
* Without deduplication, a seek to time T would fire two events (one from each
|
||||
* adapter), doubling per-scrub work in any composition that has both present.
|
||||
*
|
||||
* This module deduplicates by tracking the last dispatched time. If the same
|
||||
* time value is dispatched twice in the same synchronous call stack (e.g. two
|
||||
* adapters both calling `dispatchSeekEvent(5.0)` without yielding), only the
|
||||
* first call fires the event.
|
||||
*
|
||||
* The deduplication is intentionally coarse (exact float equality). Adapter
|
||||
* seek paths clamp and normalise time before calling this function, so the
|
||||
* values that arrive here are already stable.
|
||||
*/
|
||||
|
||||
let _lastDispatchedTime = -1;
|
||||
|
||||
export function dispatchSeekEvent(time: number): void {
|
||||
if (time === _lastDispatchedTime) return;
|
||||
_lastDispatchedTime = time;
|
||||
try {
|
||||
window.dispatchEvent(new CustomEvent("hf-seek", { detail: { time } }));
|
||||
} catch (err) {
|
||||
swallow("runtime.adapters.seek-dispatch.site1", err);
|
||||
}
|
||||
}
|
||||
|
||||
/** Reset internal state — used in tests to prevent cross-test contamination. */
|
||||
export function resetSeekDispatchState(): void {
|
||||
_lastDispatchedTime = -1;
|
||||
}
|
||||
@@ -1,11 +1,13 @@
|
||||
import { describe, it, expect, vi, beforeEach } from "vitest";
|
||||
import { createThreeAdapter } from "./three";
|
||||
import { resetSeekDispatchState } from "./seek-dispatch";
|
||||
|
||||
const threeWindow = window as Window & { __hfThreeTime?: number };
|
||||
|
||||
describe("three adapter", () => {
|
||||
beforeEach(() => {
|
||||
delete threeWindow.__hfThreeTime;
|
||||
resetSeekDispatchState();
|
||||
});
|
||||
|
||||
it("has correct name", () => {
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import type { RuntimeDeterministicAdapter } from "../types";
|
||||
import { swallow } from "../diagnostics";
|
||||
import { dispatchSeekEvent } from "./seek-dispatch";
|
||||
|
||||
export function createThreeAdapter(): RuntimeDeterministicAdapter {
|
||||
let forcedTime: number | null = null;
|
||||
@@ -12,12 +12,7 @@ export function createThreeAdapter(): RuntimeDeterministicAdapter {
|
||||
forcedTime = Math.max(0, Number(ctx.time) || 0);
|
||||
lastForcedTime = forcedTime;
|
||||
window.__hfThreeTime = forcedTime;
|
||||
try {
|
||||
window.dispatchEvent(new CustomEvent("hf-seek", { detail: { time: forcedTime } }));
|
||||
} catch (err) {
|
||||
// ignore custom event failures
|
||||
swallow("runtime.adapters.three.site1", err);
|
||||
}
|
||||
dispatchSeekEvent(forcedTime);
|
||||
},
|
||||
pause: () => {
|
||||
if (forcedTime == null) {
|
||||
|
||||
@@ -0,0 +1,89 @@
|
||||
import { describe, it, expect, vi, beforeEach } from "vitest";
|
||||
import { createTypegpuAdapter } from "./typegpu";
|
||||
import { resetSeekDispatchState } from "./seek-dispatch";
|
||||
|
||||
const gpuWindow = window as Window & { __hfTypegpuTime?: number };
|
||||
|
||||
describe("typegpu adapter", () => {
|
||||
beforeEach(() => {
|
||||
delete gpuWindow.__hfTypegpuTime;
|
||||
// Reset shared dedup state so each test starts with a clean dispatch history
|
||||
resetSeekDispatchState();
|
||||
});
|
||||
|
||||
it("has correct name", () => {
|
||||
expect(createTypegpuAdapter().name).toBe("typegpu");
|
||||
});
|
||||
|
||||
it("seek sets __hfTypegpuTime", () => {
|
||||
const adapter = createTypegpuAdapter();
|
||||
adapter.seek({ time: 5 });
|
||||
expect(gpuWindow.__hfTypegpuTime).toBe(5);
|
||||
});
|
||||
|
||||
it("seek dispatches hf-seek custom event with time", () => {
|
||||
const adapter = createTypegpuAdapter();
|
||||
const handler = vi.fn();
|
||||
window.addEventListener("hf-seek", handler);
|
||||
adapter.seek({ time: 3.5 });
|
||||
window.removeEventListener("hf-seek", handler);
|
||||
expect(handler).toHaveBeenCalledOnce();
|
||||
const detail = (handler.mock.calls[0][0] as CustomEvent).detail;
|
||||
expect(detail.time).toBe(3.5);
|
||||
});
|
||||
|
||||
it("seek clamps negative time to 0", () => {
|
||||
const adapter = createTypegpuAdapter();
|
||||
adapter.seek({ time: -5 });
|
||||
expect(gpuWindow.__hfTypegpuTime).toBe(0);
|
||||
});
|
||||
|
||||
it("seek handles NaN gracefully", () => {
|
||||
const adapter = createTypegpuAdapter();
|
||||
adapter.seek({ time: NaN });
|
||||
expect(gpuWindow.__hfTypegpuTime).toBe(0);
|
||||
});
|
||||
|
||||
it("multiple seeks to different times dispatch separate events", () => {
|
||||
const adapter = createTypegpuAdapter();
|
||||
const handler = vi.fn();
|
||||
window.addEventListener("hf-seek", handler);
|
||||
adapter.seek({ time: 1 });
|
||||
adapter.seek({ time: 2 });
|
||||
adapter.seek({ time: 3 });
|
||||
window.removeEventListener("hf-seek", handler);
|
||||
expect(handler).toHaveBeenCalledTimes(3);
|
||||
});
|
||||
|
||||
it("duplicate seek to same time fires event only once (dedup)", () => {
|
||||
const adapter = createTypegpuAdapter();
|
||||
const handler = vi.fn();
|
||||
window.addEventListener("hf-seek", handler);
|
||||
adapter.seek({ time: 5 });
|
||||
adapter.seek({ time: 5 }); // same time — deduplicated
|
||||
window.removeEventListener("hf-seek", handler);
|
||||
expect(handler).toHaveBeenCalledOnce();
|
||||
// __hfTypegpuTime is still updated on every seek regardless of dedup
|
||||
expect(gpuWindow.__hfTypegpuTime).toBe(5);
|
||||
});
|
||||
|
||||
it("pause after seek preserves last time", () => {
|
||||
const adapter = createTypegpuAdapter();
|
||||
adapter.seek({ time: 8 });
|
||||
adapter.pause();
|
||||
expect(gpuWindow.__hfTypegpuTime).toBe(8);
|
||||
});
|
||||
|
||||
it("revert resets state", () => {
|
||||
const adapter = createTypegpuAdapter();
|
||||
adapter.seek({ time: 5 });
|
||||
adapter.revert!();
|
||||
adapter.pause();
|
||||
expect(gpuWindow.__hfTypegpuTime).toBe(5);
|
||||
});
|
||||
|
||||
it("discover is a no-op and does not throw", () => {
|
||||
const adapter = createTypegpuAdapter();
|
||||
expect(() => adapter.discover()).not.toThrow();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,97 @@
|
||||
import type { RuntimeDeterministicAdapter } from "../types";
|
||||
import { dispatchSeekEvent } from "./seek-dispatch";
|
||||
|
||||
/**
|
||||
* TypeGPU / WebGPU adapter for HyperFrames
|
||||
*
|
||||
* Enables seekable GPU-rendered compositions built with TypeGPU or raw WebGPU.
|
||||
* Since WebGPU pipelines are not introspectable from outside (unlike GSAP
|
||||
* timelines or Lottie instances), this adapter uses the same push+poll pattern
|
||||
* as the Three.js adapter:
|
||||
*
|
||||
* - `window.__hfTypegpuTime` — poll this from your rAF/render loop instead
|
||||
* of `performance.now()` to get the current seek position in seconds.
|
||||
*
|
||||
* - `"hf-seek"` CustomEvent on `window` — listen for this to imperatively
|
||||
* re-render a single frame at the new seek position.
|
||||
*
|
||||
* ## Usage in a composition
|
||||
*
|
||||
* ```html
|
||||
* <canvas id="gpu-canvas" width="1920" height="1080"></canvas>
|
||||
* <script type="module">
|
||||
* const adapter = await navigator.gpu.requestAdapter();
|
||||
* const device = await adapter.requestDevice();
|
||||
* // ... build your pipeline ...
|
||||
*
|
||||
* function render(timeSeconds) {
|
||||
* // update your time uniform and submit a draw call
|
||||
* device.queue.writeBuffer(uniformBuf, 0, new Float32Array([timeSeconds]));
|
||||
* // ... submit command encoder ...
|
||||
* }
|
||||
*
|
||||
* // Seek: fired by HyperFrames whenever the player scrubs or plays
|
||||
* window.addEventListener("hf-seek", (e) => render(e.detail.time));
|
||||
*
|
||||
* // Initial frame at t=0
|
||||
* render(window.__hfTypegpuTime ?? 0);
|
||||
* </script>
|
||||
* ```
|
||||
*
|
||||
* Works with TypeGPU (https://docs.swmansion.com/TypeGPU) and raw WebGPU alike.
|
||||
* The adapter makes no assumptions about how the pipeline is constructed.
|
||||
* Multiple canvases / renderers are supported — each listens for the same event.
|
||||
*
|
||||
* ## Render-mode determinism
|
||||
*
|
||||
* For frame-perfect video renders, call `await device.queue.onSubmittedWorkDone()`
|
||||
* after each `render(time)` invocation before the frame is captured. This ensures
|
||||
* the GPU has finished writing to the canvas before the engine screenshots it.
|
||||
*
|
||||
* ## Browser feature detection
|
||||
*
|
||||
* Always guard against environments where WebGPU is unavailable:
|
||||
*
|
||||
* ```js
|
||||
* if (!navigator.gpu) { /* fallback or early return *\/ }
|
||||
* const adapter = await navigator.gpu.requestAdapter();
|
||||
* if (!adapter) { /* GPU unavailable — software fallback *\/ }
|
||||
* ```
|
||||
*
|
||||
* The adapter itself does not check for WebGPU support — that is the
|
||||
* composition author's responsibility.
|
||||
*/
|
||||
export function createTypegpuAdapter(): RuntimeDeterministicAdapter {
|
||||
let forcedTime: number | null = null;
|
||||
let lastForcedTime = 0;
|
||||
|
||||
return {
|
||||
name: "typegpu",
|
||||
|
||||
discover: () => {
|
||||
// WebGPU pipelines have no global registry — nothing to auto-discover.
|
||||
},
|
||||
|
||||
seek: (ctx) => {
|
||||
forcedTime = Math.max(0, Number(ctx.time) || 0);
|
||||
lastForcedTime = forcedTime;
|
||||
window.__hfTypegpuTime = forcedTime;
|
||||
dispatchSeekEvent(forcedTime);
|
||||
},
|
||||
|
||||
pause: () => {
|
||||
if (forcedTime == null) {
|
||||
forcedTime = Math.max(0, lastForcedTime);
|
||||
}
|
||||
},
|
||||
|
||||
play: () => {
|
||||
forcedTime = null;
|
||||
},
|
||||
|
||||
revert: () => {
|
||||
forcedTime = null;
|
||||
lastForcedTime = 0;
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
/**
|
||||
* Patches `GPUQueue.copyExternalImageToTexture` so that video-backed WebGPU
|
||||
* effects work in both preview and render mode.
|
||||
*
|
||||
* During render, the engine's video-frame injector replaces each `<video>`
|
||||
* with a pre-decoded `<img class="__render_frame__">` sibling. Chrome's
|
||||
* headless compositor can't supply decoded frames from the native `<video>`
|
||||
* element to WebGPU, so `copyExternalImageToTexture({ source: video })`
|
||||
* fails with "Browser fails extracting valid resource from external image."
|
||||
*
|
||||
* This patch checks whether a render-frame `<img>` exists next to the
|
||||
* source `<video>`. If it does and has decoded pixels, the patch
|
||||
* transparently substitutes it as the copy source. In preview mode (no
|
||||
* render-frame sibling), the original video path is used unchanged.
|
||||
*/
|
||||
export function patchVideoTextureCompat(): void {
|
||||
const GPUQueueCtor = (globalThis as Record<string, unknown>).GPUQueue as
|
||||
| { prototype: Record<string, unknown> }
|
||||
| undefined;
|
||||
|
||||
if (!GPUQueueCtor?.prototype?.copyExternalImageToTexture) return;
|
||||
|
||||
const orig = GPUQueueCtor.prototype.copyExternalImageToTexture as (
|
||||
source: unknown,
|
||||
destination: unknown,
|
||||
copySize: unknown,
|
||||
) => void;
|
||||
|
||||
GPUQueueCtor.prototype.copyExternalImageToTexture = function (
|
||||
source: Record<string, unknown>,
|
||||
destination: unknown,
|
||||
copySize: unknown,
|
||||
) {
|
||||
if (source?.source instanceof HTMLVideoElement) {
|
||||
const sibling = source.source.nextElementSibling;
|
||||
if (
|
||||
sibling instanceof HTMLImageElement &&
|
||||
sibling.classList.contains("__render_frame__") &&
|
||||
sibling.complete &&
|
||||
sibling.naturalWidth > 0
|
||||
) {
|
||||
return orig.call(this, { ...source, source: sibling }, destination, copySize);
|
||||
}
|
||||
}
|
||||
return orig.call(this, source, destination, copySize);
|
||||
};
|
||||
}
|
||||
@@ -5,6 +5,8 @@ import { createGsapAdapter } from "./adapters/gsap";
|
||||
import { createAnimeJsAdapter } from "./adapters/animejs";
|
||||
import { createLottieAdapter } from "./adapters/lottie";
|
||||
import { createThreeAdapter } from "./adapters/three";
|
||||
import { createTypegpuAdapter } from "./adapters/typegpu";
|
||||
import { patchVideoTextureCompat } from "./adapters/video-texture-compat";
|
||||
import { createWaapiAdapter } from "./adapters/waapi";
|
||||
import { refreshRuntimeMediaCache, syncRuntimeMedia } from "./media";
|
||||
import { createPickerModule } from "./picker";
|
||||
@@ -1634,8 +1636,10 @@ export function initSandboxRuntimeModular(): void {
|
||||
createAnimeJsAdapter(),
|
||||
createLottieAdapter(),
|
||||
createThreeAdapter(),
|
||||
createTypegpuAdapter(),
|
||||
createGsapAdapter({ getTimeline: () => state.capturedTimeline }),
|
||||
] as RuntimeDeterministicAdapter[];
|
||||
patchVideoTextureCompat();
|
||||
installRuntimeErrorDiagnostics();
|
||||
runAdapters("discover");
|
||||
bindMediaMetadataListeners();
|
||||
|
||||
+9
@@ -35,6 +35,15 @@ declare global {
|
||||
__HF_FPS?: number;
|
||||
__HF_MAX_DURATION_SEC?: number;
|
||||
__hfThreeTime?: number;
|
||||
/**
|
||||
* Current seek position in seconds, set by the TypeGPU/WebGPU adapter.
|
||||
* Poll this from your WebGPU render loop instead of `performance.now()`
|
||||
* to get the deterministic seek position.
|
||||
*
|
||||
* Also listen for the `"hf-seek"` CustomEvent on `window` for an
|
||||
* imperative push signal: `window.addEventListener("hf-seek", e => render(e.detail.time))`.
|
||||
*/
|
||||
__hfTypegpuTime?: number;
|
||||
__HF_PICKER_API?: HyperframePickerApi;
|
||||
gsap?: {
|
||||
timeline: (params?: { paused?: boolean }) => RuntimeTimelineLike;
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"name": "typegpu-adapter",
|
||||
"description": "Regression guard for the TypeGPU/WebGPU runtime adapter. Verifies that WebGPU fragment shaders driven by hf-seek render deterministically via window.__hfTypegpuTime + the shared seek-dispatch deduplication.",
|
||||
"tags": ["regression", "adapter"],
|
||||
"minPsnr": 30,
|
||||
"maxFrameFailures": 0,
|
||||
"minAudioCorrelation": 0,
|
||||
"maxAudioLagWindows": 1,
|
||||
"renderConfig": {
|
||||
"fps": 30,
|
||||
"workers": 1
|
||||
}
|
||||
}
|
||||
File diff suppressed because one or more lines are too long
@@ -0,0 +1,122 @@
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<head>
|
||||
<style>
|
||||
body { margin: 0; background: #080818; }
|
||||
[data-composition-id="typegpu-test"] { position: relative; overflow: hidden; font-family: "Inter", sans-serif; }
|
||||
#label { position: absolute; top: 40px; left: 60px; color: rgba(255,255,255,0.5); font-size: 14px; font-weight: 700; letter-spacing: 0.2em; text-transform: uppercase; }
|
||||
.cap { position: absolute; bottom: 100px; left: 0; right: 0; text-align: center; font-size: 72px; font-weight: 800; color: #fff; letter-spacing: -0.03em; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div id="root" data-composition-id="typegpu-test" data-width="1920" data-height="1080" data-start="0" data-duration="6">
|
||||
|
||||
<canvas id="gpu-canvas" style="position:absolute;top:0;left:0;width:1920px;height:1080px"></canvas>
|
||||
<div id="label">TypeGPU adapter test</div>
|
||||
<div id="cap-1" class="cap">WebGPU is seekable.</div>
|
||||
<div id="cap-2" class="cap">GPU rendering works.</div>
|
||||
|
||||
<script src="https://cdn.jsdelivr.net/npm/gsap@3.14.2/dist/gsap.min.js"></script>
|
||||
<script>
|
||||
window.__timelines = window.__timelines || {};
|
||||
|
||||
gsap.set('.cap', { opacity: 0 });
|
||||
var tl = gsap.timeline({ paused: true });
|
||||
tl.to('#cap-1', { opacity: 1, duration: 0.3 }, 0.5);
|
||||
tl.to('#cap-1', { opacity: 0, duration: 0.2 }, 2.8);
|
||||
tl.to('#cap-2', { opacity: 1, duration: 0.3 }, 3.2);
|
||||
tl.to('#cap-2', { opacity: 0, duration: 0.3 }, 5.5);
|
||||
window.__timelines['typegpu-test'] = tl;
|
||||
|
||||
// ── WebGPU: procedural gradient + animated ring (no video dependency) ──
|
||||
var WGSL = `
|
||||
struct U { time: f32, _p1: f32, _p2: f32, _p3: f32 }
|
||||
@group(0) @binding(0) var<uniform> u: U;
|
||||
|
||||
struct Vo { @builtin(position) pos: vec4f, @location(0) uv: vec2f }
|
||||
|
||||
@vertex fn vs(@builtin(vertex_index) vi: u32) -> Vo {
|
||||
let ps = array<vec2f,3>(vec2f(-1,-1), vec2f(3,-1), vec2f(-1,3));
|
||||
let ts = array<vec2f,3>(vec2f(0,1), vec2f(2,1), vec2f(0,-1));
|
||||
return Vo(vec4f(ps[vi],0,1), ts[vi]);
|
||||
}
|
||||
|
||||
fn hsv(h: f32, s: f32, v: f32) -> vec3f {
|
||||
let k = vec4f(1.0, 2.0/3.0, 1.0/3.0, 3.0);
|
||||
let p = abs(fract(vec3f(h) + k.xyz) * 6.0 - k.www);
|
||||
return v * mix(k.xxx, clamp(p - k.xxx, vec3f(0.0), vec3f(1.0)), s);
|
||||
}
|
||||
|
||||
@fragment fn fs(in: Vo) -> @location(0) vec4f {
|
||||
let center = vec2f(0.5, 0.5);
|
||||
let p = in.uv - center;
|
||||
let dist = length(p);
|
||||
let angle = atan2(p.y, p.x);
|
||||
|
||||
// Animated ring
|
||||
let ring = smoothstep(0.28, 0.3, dist) * (1.0 - smoothstep(0.32, 0.34, dist));
|
||||
let hue = fract(angle / 6.283 + u.time * 0.15);
|
||||
let ring_color = hsv(hue, 0.9, 1.0) * ring;
|
||||
|
||||
// Background gradient
|
||||
let bg = mix(
|
||||
vec3f(0.03, 0.03, 0.09),
|
||||
vec3f(0.08, 0.04, 0.18),
|
||||
in.uv.y
|
||||
);
|
||||
|
||||
// Pulsing center glow
|
||||
let glow = exp(-dist * dist * 18.0) * 0.3 * (0.8 + 0.2 * sin(u.time * 2.0));
|
||||
let glow_color = vec3f(0.4, 0.2, 0.9) * glow;
|
||||
|
||||
return vec4f(bg + ring_color + glow_color, 1.0);
|
||||
}`;
|
||||
|
||||
(async function() {
|
||||
if (!navigator.gpu) return;
|
||||
var adapter = await navigator.gpu.requestAdapter();
|
||||
if (!adapter) return;
|
||||
var device = await adapter.requestDevice();
|
||||
var canvas = document.getElementById('gpu-canvas');
|
||||
canvas.width = 1920; canvas.height = 1080;
|
||||
var ctx = canvas.getContext('webgpu');
|
||||
var fmt = navigator.gpu.getPreferredCanvasFormat();
|
||||
ctx.configure({ device: device, format: fmt, alphaMode: 'opaque' });
|
||||
|
||||
var uData = new Float32Array([0, 0, 0, 0]);
|
||||
var uBuf = device.createBuffer({ size: 16, usage: GPUBufferUsage.UNIFORM | GPUBufferUsage.COPY_DST });
|
||||
device.queue.writeBuffer(uBuf, 0, uData);
|
||||
|
||||
var mod = device.createShaderModule({ code: WGSL });
|
||||
var bgl = device.createBindGroupLayout({
|
||||
entries: [{ binding: 0, visibility: GPUShaderStage.VERTEX | GPUShaderStage.FRAGMENT, buffer: { type: 'uniform' } }],
|
||||
});
|
||||
var pipeline = device.createRenderPipeline({
|
||||
layout: device.createPipelineLayout({ bindGroupLayouts: [bgl] }),
|
||||
vertex: { module: mod, entryPoint: 'vs' },
|
||||
fragment: { module: mod, entryPoint: 'fs', targets: [{ format: fmt }] },
|
||||
primitive: { topology: 'triangle-list' },
|
||||
});
|
||||
var bg = device.createBindGroup({ layout: bgl, entries: [{ binding: 0, resource: { buffer: uBuf } }] });
|
||||
|
||||
function render(t) {
|
||||
uData[0] = t;
|
||||
device.queue.writeBuffer(uBuf, 0, uData);
|
||||
var enc = device.createCommandEncoder();
|
||||
var pass = enc.beginRenderPass({
|
||||
colorAttachments: [{ view: ctx.getCurrentTexture().createView(), loadOp: 'clear', clearValue: { r: 0, g: 0, b: 0, a: 1 }, storeOp: 'store' }],
|
||||
});
|
||||
pass.setPipeline(pipeline);
|
||||
pass.setBindGroup(0, bg);
|
||||
pass.draw(3);
|
||||
pass.end();
|
||||
device.queue.submit([enc.finish()]);
|
||||
}
|
||||
|
||||
render(0);
|
||||
window.addEventListener('hf-seek', function(e) { render(e.detail.time); });
|
||||
})();
|
||||
</script>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,174 @@
|
||||
---
|
||||
name: typegpu
|
||||
description: TypeGPU and raw WebGPU adapter patterns for HyperFrames. Use when creating GPU-rendered compositions with TypeGPU, raw WebGPU, WGSL fragment shaders, compute pipelines, liquid glass effects, particle systems, or any canvas layer driven by navigator.gpu that responds to HyperFrames hf-seek events.
|
||||
---
|
||||
|
||||
# TypeGPU / WebGPU for HyperFrames
|
||||
|
||||
HyperFrames supports TypeGPU and raw WebGPU through its `typegpu` runtime adapter. The adapter does not own your pipeline. It publishes HyperFrames time and dispatches a seek event so your composition can render the exact GPU frame.
|
||||
|
||||
## Contract
|
||||
|
||||
- Initialize WebGPU asynchronously (`await navigator.gpu.requestAdapter()`), but register all GSAP tweens **synchronously** — before any `await`. The HyperFrames player reads the timeline immediately at page load.
|
||||
- Render from HyperFrames time, not `performance.now()`.
|
||||
- Listen for the `hf-seek` event and re-render at exactly that time.
|
||||
- Guard against environments where WebGPU is unavailable — the adapter does not check for you.
|
||||
- For video renders, call `await device.queue.onSubmittedWorkDone()` after submitting GPU work to ensure the canvas is flushed before the frame is captured.
|
||||
|
||||
The adapter sets `window.__hfTypegpuTime` and dispatches `new CustomEvent("hf-seek", { detail: { time } })` on each seek.
|
||||
|
||||
## Basic Pattern
|
||||
|
||||
```html
|
||||
<canvas id="gpu-layer"></canvas>
|
||||
<script>
|
||||
(async () => {
|
||||
if (!navigator.gpu) return;
|
||||
const adapter = await navigator.gpu.requestAdapter();
|
||||
if (!adapter) return;
|
||||
const device = await adapter.requestDevice();
|
||||
const canvas = document.getElementById("gpu-layer");
|
||||
canvas.width = 1920;
|
||||
canvas.height = 1080;
|
||||
const ctx = canvas.getContext("webgpu");
|
||||
const fmt = navigator.gpu.getPreferredCanvasFormat();
|
||||
ctx.configure({ device, format: fmt, alphaMode: "opaque" });
|
||||
|
||||
// Build your pipeline, buffers, bind groups...
|
||||
const timeUniform = new Float32Array([0]);
|
||||
const timeBuf = device.createBuffer({
|
||||
size: 16,
|
||||
usage: GPUBufferUsage.UNIFORM | GPUBufferUsage.COPY_DST,
|
||||
});
|
||||
|
||||
function render(t) {
|
||||
timeUniform[0] = t;
|
||||
device.queue.writeBuffer(timeBuf, 0, timeUniform);
|
||||
const enc = device.createCommandEncoder();
|
||||
const pass = enc.beginRenderPass({
|
||||
colorAttachments: [
|
||||
{
|
||||
view: ctx.getCurrentTexture().createView(),
|
||||
loadOp: "clear",
|
||||
clearValue: { r: 0, g: 0, b: 0, a: 1 },
|
||||
storeOp: "store",
|
||||
},
|
||||
],
|
||||
});
|
||||
pass.setPipeline(pipeline);
|
||||
pass.setBindGroup(0, bindGroup);
|
||||
pass.draw(3);
|
||||
pass.end();
|
||||
device.queue.submit([enc.finish()]);
|
||||
}
|
||||
|
||||
render(0);
|
||||
window.addEventListener("hf-seek", (e) => render(e.detail.time));
|
||||
})();
|
||||
</script>
|
||||
```
|
||||
|
||||
## Timeline Registration
|
||||
|
||||
GSAP tweens that drive text, captions, or HTML elements must be registered **synchronously** — before any `await`:
|
||||
|
||||
```js
|
||||
const tl = gsap.timeline({ paused: true });
|
||||
|
||||
// Caption tweens: synchronous, added before WebGPU init
|
||||
gsap.set(".cap", { opacity: 0 });
|
||||
tl.to("#cap-1", { opacity: 1, duration: 0.3 }, 1.0);
|
||||
tl.to("#cap-1", { opacity: 0, duration: 0.2 }, 3.5);
|
||||
|
||||
window.__timelines["my-comp"] = tl;
|
||||
|
||||
// GPU-dependent tweens can go inside the async IIFE
|
||||
(async () => {
|
||||
// ... WebGPU init ...
|
||||
const proxy = { value: 0 };
|
||||
tl.to(proxy, { value: 1, duration: 2, onUpdate: render }, 0.5);
|
||||
})();
|
||||
```
|
||||
|
||||
## Video-Backed Effects (Liquid Glass, Distortion)
|
||||
|
||||
To use a `<video>` as the GPU input texture:
|
||||
|
||||
```js
|
||||
const videoEl = document.getElementById("aroll");
|
||||
|
||||
// Wait for video metadata before creating the texture
|
||||
await new Promise((r) => {
|
||||
if (videoEl.readyState >= 1) r();
|
||||
else videoEl.addEventListener("loadedmetadata", r, { once: true });
|
||||
});
|
||||
|
||||
// Create texture at the video's NATIVE resolution
|
||||
const vw = videoEl.videoWidth,
|
||||
vh = videoEl.videoHeight;
|
||||
const bgTex = device.createTexture({
|
||||
size: [vw, vh],
|
||||
format: "rgba8unorm",
|
||||
usage:
|
||||
GPUTextureUsage.COPY_DST | GPUTextureUsage.TEXTURE_BINDING | GPUTextureUsage.RENDER_ATTACHMENT,
|
||||
});
|
||||
|
||||
function render(t) {
|
||||
try {
|
||||
device.queue.copyExternalImageToTexture({ source: videoEl }, { texture: bgTex }, [vw, vh]);
|
||||
} catch (_) {
|
||||
/* frame not decoded yet */
|
||||
}
|
||||
// ... draw ...
|
||||
}
|
||||
```
|
||||
|
||||
**Render-mode caveat:** headless Chrome may fail `copyExternalImageToTexture` for video elements. For production renders, pre-extract key frames via FFmpeg as PNGs and load them as image textures instead.
|
||||
|
||||
## Frosted Blur via Downsample Pass
|
||||
|
||||
A single-pass Gaussian kernel is too weak for glass-like frosted blur. Use a two-pass approach:
|
||||
|
||||
1. **Pass 1 — Downsample:** render the full-res texture to a small texture (1/6 resolution). Bilinear filtering during the downsample naturally averages pixels.
|
||||
2. **Pass 2 — Glass composite:** sample the small texture for the frosted interior (bilinear upscale = heavy smooth blur) and the full-res texture for sharp areas and chromatic refraction.
|
||||
|
||||
This matches TypeGPU's `textureSampleBias` mip-level approach without generating mipmaps.
|
||||
|
||||
## Transparent vs Opaque Canvas
|
||||
|
||||
- **`alphaMode: 'opaque'`** — the GPU canvas renders the full frame (video + effect). Use when the GPU pipeline handles all visual content.
|
||||
- **`alphaMode: 'premultiplied'`** — the GPU canvas is transparent where alpha = 0, letting HTML elements below show through. Use for overlays (particles, path animations) on top of a regular `<video>` element.
|
||||
|
||||
## WGSL Full-Screen Triangle
|
||||
|
||||
The standard vertex shader for full-screen effects (no vertex buffer needed):
|
||||
|
||||
```wgsl
|
||||
struct Vo { @builtin(position) pos: vec4f, @location(0) uv: vec2f }
|
||||
|
||||
@vertex fn vs(@builtin(vertex_index) vi: u32) -> Vo {
|
||||
let ps = array<vec2f, 3>(vec2f(-1., -1.), vec2f(3., -1.), vec2f(-1., 3.));
|
||||
let ts = array<vec2f, 3>(vec2f(0., 1.), vec2f(2., 1.), vec2f(0., -1.));
|
||||
return Vo(vec4f(ps[vi], 0., 1.), ts[vi]);
|
||||
}
|
||||
```
|
||||
|
||||
Draw with `pass.draw(3)` — one triangle that covers the viewport.
|
||||
|
||||
## Rounded-Rect SDF (Liquid Glass Pill)
|
||||
|
||||
```wgsl
|
||||
fn sdf_box(p: vec2f, half_size: vec2f, corner_radius: f32) -> f32 {
|
||||
let d = abs(p) - half_size + vec2f(corner_radius);
|
||||
return length(max(d, vec2f(0.))) + min(max(d.x, d.y), 0.) - corner_radius;
|
||||
}
|
||||
```
|
||||
|
||||
Use this to define inside/ring/outside zones for glass effects. Negative values are inside the shape.
|
||||
|
||||
## Deterministic Rendering
|
||||
|
||||
- No `Math.random()` — use a seeded PRNG.
|
||||
- No `requestAnimationFrame` for the render loop — render only in response to `hf-seek`.
|
||||
- No `performance.now()` for animation time — read `window.__hfTypegpuTime` or `e.detail.time`.
|
||||
- After GPU submit, call `await device.queue.onSubmittedWorkDone()` for render-mode frame capture.
|
||||
Reference in New Issue
Block a user