From 21ec5f800a1f5e2588ffd4bc4b4a1dd320171b0e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81ngel?= Date: Wed, 6 May 2026 00:00:08 +0200 Subject: [PATCH 01/13] fix(cli): use size-adaptive timeouts for publish uploads (#635) ## Summary - Replace the flat 30s upload timeout with a size-adaptive calculation: `max(120s, bytes / 500KB/s)` - Metadata requests (presigned URL, complete) keep the original 30s timeout - Companion to the backend change removing the 64 MB upload limit in experiment-framework ## Context With the backend size limit removed, large projects (78 MB+) need proportionally longer to upload. A 78 MB project now gets ~164s, a 500 MB project ~17 min. The old 30s timeout would abort any upload over ~15 MB on a typical connection. ## Test plan - [x] All 4 existing vitest tests pass - [x] Build succeeds, no type errors - [x] Lint + format pass (oxlint + oxfmt) - [x] Timeout values verified for 10/78/200/500/1000 MB archives --- packages/cli/src/utils/publishProject.test.ts | 17 +++++++++++ packages/cli/src/utils/publishProject.ts | 28 +++++++++++++++---- 2 files changed, 40 insertions(+), 5 deletions(-) diff --git a/packages/cli/src/utils/publishProject.test.ts b/packages/cli/src/utils/publishProject.test.ts index b5a3b8117..4764e0409 100644 --- a/packages/cli/src/utils/publishProject.test.ts +++ b/packages/cli/src/utils/publishProject.test.ts @@ -7,6 +7,7 @@ import { createPublishArchive, getPublishApiBaseUrl, publishProjectArchive, + uploadTimeoutMs, } from "./publishProject.js"; function makeProjectDir(): string { @@ -35,6 +36,22 @@ describe("createPublishArchive", () => { }); }); +describe("uploadTimeoutMs", () => { + it("returns the minimum timeout for small files", () => { + expect(uploadTimeoutMs(0)).toBe(120_000); + expect(uploadTimeoutMs(50 * 1024 * 1024)).toBe(120_000); + }); + + it("scales above the floor for large files", () => { + expect(uploadTimeoutMs(64 * 1024 * 1024)).toBeGreaterThan(120_000); + expect(uploadTimeoutMs(500 * 1024 * 1024)).toBeGreaterThan(900_000); + }); + + it("returns an integer", () => { + expect(Number.isInteger(uploadTimeoutMs(123_456))).toBe(true); + }); +}); + describe("publishProjectArchive", () => { beforeEach(() => { vi.stubEnv("HYPERFRAMES_PUBLISHED_PROJECTS_API_URL", ""); diff --git a/packages/cli/src/utils/publishProject.ts b/packages/cli/src/utils/publishProject.ts index 3b4d62855..9a317b619 100644 --- a/packages/cli/src/utils/publishProject.ts +++ b/packages/cli/src/utils/publishProject.ts @@ -5,7 +5,11 @@ import AdmZip from "adm-zip"; const IGNORED_DIRS = new Set([".git", "node_modules", "dist", ".next", "coverage"]); const IGNORED_FILES = new Set([".DS_Store", "Thumbs.db"]); const PUBLISH_CONTENT_TYPE = "application/zip"; -const PUBLISH_REQUEST_TIMEOUT_MS = 30_000; +const PUBLISH_METADATA_TIMEOUT_MS = 30_000; +const PUBLISH_UPLOAD_MIN_TIMEOUT_MS = 120_000; +// Conservative floor — most connections are faster, but this prevents +// premature aborts on slow/unstable networks (hotel wifi, tethering). +const PUBLISH_UPLOAD_BYTES_PER_SECOND = 500_000; export interface PublishArchiveResult { buffer: Buffer; @@ -25,6 +29,7 @@ interface StagedUploadResponse { uploadKey: string; contentType: string; uploadHeaders: Record; + expiresInSeconds: number; } type JsonRecord = Record; @@ -73,11 +78,14 @@ function parseStagedUploadResponse( const uploadKey = stringField(data, "upload_key"); const contentType = stringField(data, "content_type") || PUBLISH_CONTENT_TYPE; if (!uploadUrl || !uploadKey) return null; + const rawExpires = data["expires_in_seconds"]; + const expiresInSeconds = typeof rawExpires === "number" && rawExpires > 0 ? rawExpires : 1800; return { uploadUrl, uploadKey, contentType, uploadHeaders: getUploadHeaders(data, uploadUrl, contentType, archiveByteLength), + expiresInSeconds, }; } @@ -142,6 +150,13 @@ async function readErrorMessage(response: Response, fallback: string): Promise Date: Tue, 5 May 2026 16:47:30 -0700 Subject: [PATCH 02/13] feat(cli): add --background-output to remove-background MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Emit an inverse-alpha background plate alongside the cutout in a single inference pass. Same source RGB, alpha = 255 − mask. Dual-encoder pipeline runs in parallel; both outputs share the same --quality preset. This is a hole-cut plate (subject region transparent), not an inpainted clean plate — composite something opaque under it to fill the hole. Docs and skill cover when each is the right tool. Co-Authored-By: Claude Opus 4.7 (1M context) --- docs/guides/remove-background.mdx | 65 +++++++ docs/packages/cli.mdx | 7 +- .../cli/src/background-removal/inference.ts | 69 +++++-- .../src/background-removal/pipeline.test.ts | 56 +++++- .../cli/src/background-removal/pipeline.ts | 171 +++++++++++++----- .../cli/src/commands/remove-background.ts | 21 ++- skills/hyperframes-media/SKILL.md | 55 ++++++ 7 files changed, 384 insertions(+), 60 deletions(-) diff --git a/docs/guides/remove-background.mdx b/docs/guides/remove-background.mdx index a7cb06bc7..334bb1163 100644 --- a/docs/guides/remove-background.mdx +++ b/docs/guides/remove-background.mdx @@ -80,6 +80,71 @@ npx hyperframes remove-background subject.mp4 -o transparent.mov # editi npx hyperframes remove-background portrait.jpg -o cutout.png # still image ``` +## Layer separation: emit the cutout and the background plate together + +Pass `--background-output` (alias `-b`) to write a *second* transparent video alongside the cutout. Same source RGB, alpha is the *inverse* mask — opaque where the surroundings were, transparent where the subject is. The result is a clean two-layer separation in a single inference pass: + +```bash Terminal +npx hyperframes remove-background subject.mp4 \ + -o subject.webm \ + --background-output plate.webm +``` + +| Output | Alpha | Use it as | +| ------ | ----- | --------- | +| `subject.webm` | Mask — subject opaque | Foreground layer (top of stack) | +| `plate.webm` | `255 − mask` — subject region transparent | Background layer; place anything you want **under the subject's silhouette** between this and `subject.webm` | + +Both encoders share the source W/H/fps and your `--quality` preset, so the layers are pixel-aligned. Encode cost roughly doubles; segmentation cost is unchanged. + + +**This is a hole-cut plate, not an inpainted clean plate.** The subject region in `plate.webm` is fully transparent — you have to composite something opaque under it (a graphic, a blurred copy, a different scene) to fill the hole. If you need an actual filled background where the subject was, use a video inpainter (LaMa, ProPainter, RunwayML Inpaint) — `remove-background` is not the right tool for that. + + +### Hole-cut vs. clean plate — when does the difference matter? + +A **hole-cut plate** keeps the original surroundings and makes the subject region transparent. A **clean plate** fills the subject region with reconstructed background — produced by a separate inpainting model. Display each alone over black: + +| | Hole-cut plate (this command) | Clean plate (inpainted) | +| --- | --- | --- | +| Subject region | Transparent silhouette | Reconstructed background pixels | +| What you see alone | A person-shaped hole | An empty room | +| Cost | One inference pass, one extra ffmpeg encode | A second model (LaMa, ProPainter, E2FGVI) | +| Tool | `remove-background --background-output` | Outside this CLI | + +The line is: **does anything ever need to be visible *through* the subject's silhouette where the subject used to be?** + +| Use case | What you need | +| --- | --- | +| Text/graphics live *between* the cutout and the plate (the example above) | **Hole-cut** — the graphics fill the hole. | +| Composite the subject onto an unrelated scene | Neither. Just use `subject.webm`; the plate is irrelevant. | +| Show "the room without the person" as a real background | **Clean plate** — a hole-cut plate would show a transparent void. | +| Replace the person with a different subject (re-target) | **Clean plate** — the new subject needs real pixels under it. | +| VFX rotoscoping / "remove an extra from this take" | **Clean plate** — the canonical inpainting use case. | + +If something opaque always covers the silhouette, hole-cut is sufficient and ~1000× cheaper than running an inpainter. + +### The two-layer composition pattern + +The two-layer pattern is functionally a drop-in for [text-behind-subject](#text-behind-subject-the-recommended-layout) without needing the original `presenter.mp4` in the project — the plate replaces it as the bottom layer: + +```html + + + + +

+ MAKE IT IN HYPERFRAMES +

+ + +
+ +
+``` + +Constraints: the flag requires a video input and `.webm` or `.mov` for both outputs. It's not valid for image inputs (no temporal pairing to do) and won't accept `.png` for the plate. + ## Performance Real-world numbers from the [matting eval](https://www.heygenverse.com/a/0dd5a431-1832-4858-862d-de7fb7d02654), running u²-net_human_seg on a 4-second 1080p clip: diff --git a/docs/packages/cli.mdx b/docs/packages/cli.mdx index 57f8a0811..f9ae12f72 100644 --- a/docs/packages/cli.mdx +++ b/docs/packages/cli.mdx @@ -356,6 +356,10 @@ This is suppressed in CI environments, non-TTY shells, and when `HYPERFRAMES_NO_ # Single image → transparent PNG npx hyperframes remove-background portrait.jpg -o cutout.png + # Layer separation: cutout AND inverse-alpha background plate in one pass + npx hyperframes remove-background avatar.mp4 \ + -o subject.webm --background-output plate.webm + # Force CPU on a machine that has CoreML or CUDA npx hyperframes remove-background avatar.mp4 -o transparent.webm --device cpu @@ -366,8 +370,9 @@ This is suppressed in CI environments, non-TTY shells, and when `HYPERFRAMES_NO_ | Flag | Description | |------|-------------| | `--output, -o` | Output path. Format inferred from extension: `.webm` (default), `.mov`, `.png` | + | `--background-output, -b` | Optional second output: inverse-alpha background plate (subject region transparent, surroundings opaque). Same source RGB, complementary mask. Must be `.webm` or `.mov`. Hole-cut, not inpainted — composite something underneath to fill the hole. | | `--device` | Execution provider: `auto` (default), `cpu`, `coreml`, `cuda` | - | `--quality` | WebM encoder preset: `fast` (crf 30, smallest), `balanced` (crf 18, default), `best` (crf 12, near-lossless). Higher quality keeps the cutout's RGB closer to the source mp4 — important when overlaying the cutout on its own source for text-behind-subject effects. Ignored for `.mov` / `.png`. | + | `--quality` | WebM encoder preset: `fast` (crf 30, smallest), `balanced` (crf 18, default), `best` (crf 12, near-lossless). Higher quality keeps the cutout's RGB closer to the source mp4 — important when overlaying the cutout on its own source for text-behind-subject effects. Applies to both `--output` and `--background-output`. Ignored for `.mov` / `.png`. | | `--info` | Print detected execution providers and exit (no render) | | `--json` | Output result as JSON | diff --git a/packages/cli/src/background-removal/inference.ts b/packages/cli/src/background-removal/inference.ts index d0b3d8adb..614038663 100644 --- a/packages/cli/src/background-removal/inference.ts +++ b/packages/cli/src/background-removal/inference.ts @@ -24,10 +24,24 @@ interface OrtModule { Tensor: typeof Tensor; } +export interface SessionResult { + /** Subject opaque, background fully transparent. */ + fg: Buffer; + /** Inverse-alpha plate: same RGB, alpha is `255 − mask`. Null unless `withBackground` was true. */ + bg: Buffer | null; +} + export interface Session { - /** Run inference on one RGB frame, return RGBA bytes (H*W*4). */ - process(rgb: Buffer, width: number, height: number): Promise; - /** ORT EP that was actually selected. */ + /** + * Both `fg` and `bg` (when requested) are session-owned buffers reused on the + * next call — drain the encoder's stdin before invoking `process` again. + */ + process( + rgb: Buffer, + width: number, + height: number, + withBackground?: boolean, + ): Promise; provider: string; close(): Promise; } @@ -73,16 +87,15 @@ export async function createSession(options: CreateSessionOptions = {}): Promise throw new Error("ONNX session is missing input or output bindings"); } - // Pre-allocated per-frame buffers reused across every process() call. - // At 1080p this saves ~9 MB of allocations per frame. rgbaBuf is sized - // lazily on the first call (we don't know W/H until then). + // Reused across calls; sized lazily on first frame. Saves ~9 MB/frame at 1080p. const inputData = new Float32Array(3 * INPUT_PLANE); const maskBuf = Buffer.allocUnsafe(INPUT_PLANE); let rgbaBuf: Buffer | null = null; + let rgbaBgBuf: Buffer | null = null; return { provider: providerUsed, - async process(rgb, width, height) { + async process(rgb, width, height, withBackground = false) { const tensor = await preprocess(sharp, ort, rgb, width, height, inputData); const outputs = await session.run({ [inputName]: tensor }); const output = outputs[outputName]; @@ -91,7 +104,21 @@ export async function createSession(options: CreateSessionOptions = {}): Promise if (!rgbaBuf || rgbaBuf.length !== expectedBytes) { rgbaBuf = Buffer.allocUnsafe(expectedBytes); } - return await postprocess(sharp, output, rgb, width, height, maskBuf, rgbaBuf); + if (withBackground) { + if (!rgbaBgBuf || rgbaBgBuf.length !== expectedBytes) { + rgbaBgBuf = Buffer.allocUnsafe(expectedBytes); + } + } + return await postprocess( + sharp, + output, + rgb, + width, + height, + maskBuf, + rgbaBuf, + withBackground ? rgbaBgBuf : null, + ); }, async close() { await session.release(); @@ -141,7 +168,8 @@ async function postprocess( height: number, maskBuf: Buffer, rgbaBuf: Buffer, -): Promise { + rgbaBgBuf: Buffer | null, +): Promise { const raw = output.data as Float32Array; let lo = Infinity; @@ -172,11 +200,30 @@ async function postprocess( .raw() .toBuffer(); - for (let i = 0; i < width * height; i++) { + const pixels = width * height; + if (rgbaBgBuf) { + for (let i = 0; i < pixels; i++) { + const r = rgb[i * 3]!; + const g = rgb[i * 3 + 1]!; + const b = rgb[i * 3 + 2]!; + const m = fullMask[i]!; + const o = i * 4; + rgbaBuf[o] = r; + rgbaBuf[o + 1] = g; + rgbaBuf[o + 2] = b; + rgbaBuf[o + 3] = m; + rgbaBgBuf[o] = r; + rgbaBgBuf[o + 1] = g; + rgbaBgBuf[o + 2] = b; + rgbaBgBuf[o + 3] = 255 - m; + } + return { fg: rgbaBuf, bg: rgbaBgBuf }; + } + for (let i = 0; i < pixels; i++) { rgbaBuf[i * 4] = rgb[i * 3]!; rgbaBuf[i * 4 + 1] = rgb[i * 3 + 1]!; rgbaBuf[i * 4 + 2] = rgb[i * 3 + 2]!; rgbaBuf[i * 4 + 3] = fullMask[i]!; } - return rgbaBuf; + return { fg: rgbaBuf, bg: null }; } diff --git a/packages/cli/src/background-removal/pipeline.test.ts b/packages/cli/src/background-removal/pipeline.test.ts index 7bc5488a9..5763efb06 100644 --- a/packages/cli/src/background-removal/pipeline.test.ts +++ b/packages/cli/src/background-removal/pipeline.test.ts @@ -1,7 +1,13 @@ import { describe, expect, it } from "vitest"; import { EventEmitter } from "node:events"; import type { spawn } from "node:child_process"; -import { inferOutputFormat, inferInputKind, buildEncoderArgs, waitForExit } from "./pipeline.js"; +import { + inferOutputFormat, + inferInputKind, + buildEncoderArgs, + resolveRenderTargets, + waitForExit, +} from "./pipeline.js"; describe("background-removal/pipeline — inferOutputFormat", () => { it("maps .webm → webm", () => { @@ -97,6 +103,54 @@ describe("background-removal/pipeline — buildEncoderArgs", () => { }); }); +describe("background-removal/pipeline — resolveRenderTargets", () => { + it("resolves a normal video → webm render", () => { + const t = resolveRenderTargets("/tmp/clip.mp4", "/tmp/cutout.webm"); + expect(t.format).toBe("webm"); + expect(t.inputKind).toBe("video"); + expect(t.bgFormat).toBeUndefined(); + }); + + it("resolves an image → png render", () => { + const t = resolveRenderTargets("/tmp/portrait.jpg", "/tmp/cutout.png"); + expect(t.format).toBe("png"); + expect(t.inputKind).toBe("image"); + }); + + it("rejects image input with a video output extension", () => { + expect(() => resolveRenderTargets("/tmp/portrait.jpg", "/tmp/cutout.webm")).toThrow( + /Image input requires a \.png output/, + ); + }); + + it("rejects video input with a .png output", () => { + expect(() => resolveRenderTargets("/tmp/clip.mp4", "/tmp/cutout.png")).toThrow( + /Video input requires a \.webm or \.mov output/, + ); + }); + + it("threads background-output format through when valid", () => { + const t = resolveRenderTargets("/tmp/clip.mp4", "/tmp/fg.webm", "/tmp/bg.webm"); + expect(t.bgFormat).toBe("webm"); + const tMov = resolveRenderTargets("/tmp/clip.mp4", "/tmp/fg.webm", "/tmp/bg.mov"); + expect(tMov.bgFormat).toBe("mov"); + }); + + it("rejects --background-output for image inputs (no temporal pairing to do)", () => { + expect(() => + resolveRenderTargets("/tmp/portrait.jpg", "/tmp/cutout.png", "/tmp/bg.png"), + ).toThrow(/--background-output is not supported for image inputs/); + }); + + it("rejects .png as the --background-output extension", () => { + // .png is only valid for single-image inputs, and image inputs themselves + // can't have a background-output anyway. So .png here is always a misuse. + expect(() => resolveRenderTargets("/tmp/clip.mp4", "/tmp/fg.webm", "/tmp/bg.png")).toThrow( + /--background-output must be \.webm or \.mov/, + ); + }); +}); + // Regression: a previous version of waitForExit treated `code === null` as // success. Per Node's child_process docs, that's the signal-killed case — // reporting it as success means a SIGTERM/SIGKILL'd ffmpeg encoder produces diff --git a/packages/cli/src/background-removal/pipeline.ts b/packages/cli/src/background-removal/pipeline.ts index 99b95386e..cbd60a798 100644 --- a/packages/cli/src/background-removal/pipeline.ts +++ b/packages/cli/src/background-removal/pipeline.ts @@ -38,6 +38,16 @@ export const isQuality = (v: unknown): v is Quality => export interface RenderOptions { inputPath: string; outputPath: string; + /** + * Optional second output: an inverse-alpha background plate (same source + * RGB, transparent where the subject was). Only valid for video inputs and + * .webm/.mov outputs — not allowed alongside a .png output. The plate's + * format is inferred from this path independently of the foreground's. + * + * NOTE: this is a hole-cut plate, not an inpainted clean plate. Composite + * something opaque (graphics, blur, scene) under it to fill the hole. + */ + backgroundOutputPath?: string; device?: Device; model?: ModelId; /** Encoder CRF preset for `.webm`. See `QUALITY_CRF`. Ignored for `.mov`/`.png`. */ @@ -52,6 +62,8 @@ export type ProgressEvent = export interface RenderResult { outputPath: string; + /** Present only when `backgroundOutputPath` was set. */ + backgroundOutputPath?: string; framesProcessed: number; durationSeconds: number; avgMsPerFrame: number; @@ -203,17 +215,27 @@ async function* readFrames( } } -export async function render(options: RenderOptions): Promise { - if (!hasFFmpeg() || !hasFFprobe()) { - throw new Error("ffmpeg and ffprobe are required. Install: brew install ffmpeg"); - } +export interface RenderTargets { + format: OutputFormat; + inputKind: "video" | "image"; + bgFormat: OutputFormat | undefined; +} - const format = inferOutputFormat(options.outputPath); - const inputKind = inferInputKind(options.inputPath); +/** + * Resolve and validate the input/output combination before any I/O. Pure; + * exported so unit tests can pin the error messages without spawning ffmpeg. + */ +export function resolveRenderTargets( + inputPath: string, + outputPath: string, + backgroundOutputPath?: string, +): RenderTargets { + const format = inferOutputFormat(outputPath); + const inputKind = inferInputKind(inputPath); if (inputKind === "image" && format !== "png") { throw new Error( - `Image input requires a .png output (got ${extname(options.outputPath)}). Use a video input for .webm/.mov.`, + `Image input requires a .png output (got ${extname(outputPath)}). Use a video input for .webm/.mov.`, ); } if (inputKind === "video" && format === "png") { @@ -222,6 +244,35 @@ export async function render(options: RenderOptions): Promise { ); } + let bgFormat: OutputFormat | undefined; + if (backgroundOutputPath) { + if (inputKind === "image") { + throw new Error( + "--background-output is not supported for image inputs. Use a video input (mp4/mov/webm) to produce both a cutout and a background plate.", + ); + } + bgFormat = inferOutputFormat(backgroundOutputPath); + if (bgFormat === "png") { + throw new Error( + "--background-output must be .webm or .mov; .png is only valid for single-image inputs.", + ); + } + } + + return { format, inputKind, bgFormat }; +} + +export async function render(options: RenderOptions): Promise { + if (!hasFFmpeg() || !hasFFprobe()) { + throw new Error("ffmpeg and ffprobe are required. Install: brew install ffmpeg"); + } + + const { format, bgFormat } = resolveRenderTargets( + options.inputPath, + options.outputPath, + options.backgroundOutputPath, + ); + const media = await probeMedia(options.inputPath); options.onProgress?.({ @@ -240,12 +291,13 @@ export async function render(options: RenderOptions): Promise { try { const start = Date.now(); - const framesProcessed = await runPipeline(options, session, media, format); + const framesProcessed = await runPipeline(options, session, media, format, bgFormat); const durationSeconds = (Date.now() - start) / 1000; const avgMsPerFrame = framesProcessed ? (durationSeconds * 1000) / framesProcessed : 0; return { outputPath: options.outputPath, + backgroundOutputPath: options.backgroundOutputPath, framesProcessed, durationSeconds, avgMsPerFrame, @@ -259,61 +311,68 @@ export async function render(options: RenderOptions): Promise { const RECENT_WINDOW = 30; +interface FfmpegProc { + proc: ReturnType; + exit: Promise; + /** Tail of stderr, captured for inclusion in error messages. */ + getStderr: () => string; +} + +function spawnFfmpeg(args: string[], label: string, stdio: ("ignore" | "pipe")[]): FfmpegProc { + const proc = spawn("ffmpeg", args, { stdio }); + let stderrBuf = ""; + proc.stderr?.on("data", (d: Buffer) => { + stderrBuf += d.toString(); + }); + const exit = waitForExit(proc, label, () => stderrBuf); + return { proc, exit, getStderr: () => stderrBuf }; +} + async function runPipeline( options: RenderOptions, session: Session, media: MediaInfo, format: OutputFormat, + bgFormat: OutputFormat | undefined, ): Promise { - const { inputPath, outputPath } = options; + const { inputPath, outputPath, backgroundOutputPath } = options; const { width, height, fps, frameCount } = media; const frameBytes = width * height * 3; + const quality = options.quality ?? DEFAULT_QUALITY; - const decoder = spawn( - "ffmpeg", + const decoder = spawnFfmpeg( ["-loglevel", "error", "-i", inputPath, "-f", "rawvideo", "-pix_fmt", "rgb24", "-an", "-"], - { stdio: ["ignore", "pipe", "pipe"] }, + "ffmpeg decoder", + ["ignore", "pipe", "pipe"], ); - let decoderStderr = ""; - decoder.stderr?.on("data", (d: Buffer) => { - decoderStderr += d.toString(); - }); - const decoderExit = waitForExit(decoder, "ffmpeg decoder", () => decoderStderr); - - const encoder = spawn( - "ffmpeg", - buildEncoderArgs( - format, - width, - height, - fps || 30, - outputPath, - options.quality ?? DEFAULT_QUALITY, - ), - { - stdio: ["pipe", "ignore", "pipe"], - }, + const fg = spawnFfmpeg( + buildEncoderArgs(format, width, height, fps || 30, outputPath, quality), + "ffmpeg encoder", + ["pipe", "ignore", "pipe"], ); - let encoderStderr = ""; - encoder.stderr?.on("data", (d: Buffer) => { - encoderStderr += d.toString(); - }); - const encoderExit = waitForExit(encoder, "ffmpeg encoder", () => encoderStderr); + + const bg = + backgroundOutputPath && bgFormat + ? spawnFfmpeg( + buildEncoderArgs(bgFormat, width, height, fps || 30, backgroundOutputPath, quality), + "ffmpeg background encoder", + ["pipe", "ignore", "pipe"], + ) + : null; let processed = 0; const total = frameCount; - // Running average over the last RECENT_WINDOW frames. const recentMs = new Array(RECENT_WINDOW).fill(0); let recentSum = 0; let recentSlot = 0; let recentCount = 0; try { - for await (const rgb of readFrames(decoder.stdout!, frameBytes)) { + for await (const rgb of readFrames(decoder.proc.stdout!, frameBytes)) { const t0 = Date.now(); - const rgba = await session.process(rgb, width, height); + const result = await session.process(rgb, width, height, bg !== null); const elapsed = Date.now() - t0; recentSum += elapsed - recentMs[recentSlot]!; @@ -321,8 +380,24 @@ async function runPipeline( recentSlot = (recentSlot + 1) % RECENT_WINDOW; if (recentCount < RECENT_WINDOW) recentCount++; - if (!encoder.stdin!.write(rgba)) { - await new Promise((resolve) => encoder.stdin!.once("drain", () => resolve())); + // Issue both writes before any await so a slow encoder doesn't block + // the other. Drain anything that returned false before the next + // session.process() — its output buffers are reused per frame. + const fgWroteFully = fg.proc.stdin!.write(result.fg); + const bgWroteFully = bg && result.bg ? bg.proc.stdin!.write(result.bg) : true; + if (!fgWroteFully || !bgWroteFully) { + const drains: Promise[] = []; + if (!fgWroteFully) { + drains.push( + new Promise((resolve) => fg.proc.stdin!.once("drain", () => resolve())), + ); + } + if (!bgWroteFully && bg) { + drains.push( + new Promise((resolve) => bg.proc.stdin!.once("drain", () => resolve())), + ); + } + await Promise.all(drains); } processed++; @@ -334,17 +409,21 @@ async function runPipeline( }); } } catch (err) { - decoder.kill("SIGKILL"); - encoder.kill("SIGKILL"); + decoder.proc.kill("SIGKILL"); + fg.proc.kill("SIGKILL"); + bg?.proc.kill("SIGKILL"); throw err; } - encoder.stdin!.end(); - await Promise.all([decoderExit, encoderExit]); + fg.proc.stdin!.end(); + bg?.proc.stdin!.end(); + const exits: Promise[] = [decoder.exit, fg.exit]; + if (bg) exits.push(bg.exit); + await Promise.all(exits); if (processed === 0) { throw new Error( - `No frames produced from ${inputPath}. Decoder stderr:\n${decoderStderr.slice(-400)}`, + `No frames produced from ${inputPath}. Decoder stderr:\n${decoder.getStderr().slice(-400)}`, ); } diff --git a/packages/cli/src/commands/remove-background.ts b/packages/cli/src/commands/remove-background.ts index cc942f67e..1116bef74 100644 --- a/packages/cli/src/commands/remove-background.ts +++ b/packages/cli/src/commands/remove-background.ts @@ -20,6 +20,10 @@ export const examples: Example[] = [ "Remove background from a single image, output transparent PNG", "hyperframes remove-background portrait.jpg -o cutout.png", ], + [ + "Separate the layers — emit both the cutout and an inverse-alpha background plate (subject region transparent)", + "hyperframes remove-background avatar.mp4 -o subject.webm --background-output plate.webm", + ], [ "Force CPU (skip CoreML/CUDA)", "hyperframes remove-background avatar.mp4 -o transparent.webm --device cpu", @@ -52,6 +56,12 @@ export default defineCommand({ description: "Output path. Format inferred from extension: .webm (default), .mov, .png", alias: "o", }, + "background-output": { + type: "string", + description: + "Optional second output path for the inverse-alpha background plate (subject region transparent, original surroundings opaque). Hole-cut, not inpainted — composite something underneath to fill the hole. Must be .webm or .mov; not allowed for image inputs.", + alias: "b", + }, device: { type: "string", description: `Execution provider: ${DEVICES.join(", ")}`, @@ -104,6 +114,8 @@ export default defineCommand({ const inputPath = resolve(args.input); const outputPath = resolve(args.output); + const backgroundOutputArg = args["background-output"]; + const backgroundOutputPath = backgroundOutputArg ? resolve(backgroundOutputArg) : undefined; const { render } = await import("../background-removal/pipeline.js"); @@ -114,6 +126,7 @@ export default defineCommand({ const result = await render({ inputPath, outputPath, + backgroundOutputPath, device: args.device, quality: args.quality, onProgress: (event) => { @@ -137,6 +150,9 @@ export default defineCommand({ JSON.stringify({ ok: true, outputPath: result.outputPath, + ...(result.backgroundOutputPath + ? { backgroundOutputPath: result.backgroundOutputPath } + : {}), framesProcessed: result.framesProcessed, durationSeconds: Number(result.durationSeconds.toFixed(2)), avgMsPerFrame: Number(result.avgMsPerFrame.toFixed(1)), @@ -148,9 +164,12 @@ export default defineCommand({ const fpsThroughput = result.durationSeconds ? (result.framesProcessed / result.durationSeconds).toFixed(1) : "n/a"; + const outputs = result.backgroundOutputPath + ? `${c.accent(result.outputPath)} + ${c.accent(result.backgroundOutputPath)}` + : c.accent(result.outputPath); spin?.stop( c.success( - `Removed background from ${c.accent(String(result.framesProcessed))} frames in ${result.durationSeconds.toFixed(1)}s (${fpsThroughput} fps, ${c.accent(result.provider)}) → ${c.accent(result.outputPath)}`, + `Removed background from ${c.accent(String(result.framesProcessed))} frames in ${result.durationSeconds.toFixed(1)}s (${fpsThroughput} fps, ${c.accent(result.provider)}) → ${outputs}`, ), ); } diff --git a/skills/hyperframes-media/SKILL.md b/skills/hyperframes-media/SKILL.md index 40b1b688d..13e3d4ae0 100644 --- a/skills/hyperframes-media/SKILL.md +++ b/skills/hyperframes-media/SKILL.md @@ -108,12 +108,67 @@ Remove the background from a video or image so the subject (typically a person npx hyperframes remove-background subject.mp4 -o transparent.webm # default: VP9 alpha WebM npx hyperframes remove-background subject.mp4 -o transparent.mov # ProRes 4444 (editing) npx hyperframes remove-background portrait.jpg -o cutout.png # single-image cutout +npx hyperframes remove-background subject.mp4 -o subject.webm \ + --background-output plate.webm # both layers in one pass npx hyperframes remove-background subject.mp4 -o transparent.webm --device cpu npx hyperframes remove-background --info # detected providers ``` Uses `u2net_human_seg` (MIT). First run downloads ~168 MB of weights to `~/.cache/hyperframes/background-removal/models/`. +### Layer separation (`--background-output`) + +Pass `--background-output` (or `-b`) to emit a **second** transparent video alongside the cutout: same source RGB, alpha is `255 − mask` instead of `mask`. The cutout is the subject with a transparent background; the plate is the original surroundings with a transparent hole where the subject was. + +| File | Alpha is… | Use it for | +| -------------------------------- | --------------------------------------------------------- | --------------------------------------------------------------- | +| `-o subject.webm` | The mask — subject opaque, background transparent | Foreground layer, place on top | +| `--background-output plate.webm` | Inverse — surroundings opaque, subject region transparent | Bottom layer; put text or graphics between this and the subject | + +Both outputs share the same `--quality` preset and run from a single inference pass — encode cost roughly doubles, segmentation cost stays the same. Only valid for video inputs and `.webm`/`.mov` outputs. + +**Hole-cut plate, not an inpainted clean plate.** The subject region in `plate.webm` is fully transparent — composite something opaque under it to fill the hole. The single test for whether `--background-output` is the right tool: _will anything ever be visible through the subject's silhouette where the subject used to be?_ + +| Use case | Right tool | +| ----------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------- | +| Text/graphics between the cutout and the plate (this command's reason for existing) | **Hole-cut** (`--background-output`) | +| Subject onto an unrelated scene | Just `subject.webm`; ignore the plate | +| Show the room _without_ the person, alone over no other content | **Clean plate** — needs an inpainter (LaMa, ProPainter, E2FGVI). Not this command. | +| Replace the subject with a different subject | **Clean plate** — same as above | + +If a user asks for "the room with the person removed" and intends to display it standalone, do **not** reach for `--background-output`. Tell them they need an inpainter. + +Typical layered composition (the canonical hole-cut use case): + +```html + + + + +

MAKE IT IN HYPERFRAMES

+ + +
+ +
+``` + +This is functionally equivalent to the text-behind-subject pattern below, but you don't need the original `presenter.mp4` in the project — the plate replaces it. Useful when you want to ship just the two transparent layers and let the user drop arbitrary content between them. + ### Output Format | Format | When | From 8700826518d60899ca3ce24359faa7f1f47932c3 Mon Sep 17 00:00:00 2001 From: Rames Jusso Date: Wed, 6 May 2026 00:07:49 +0000 Subject: [PATCH 03/13] fix(lint): rephrase too-large composition warnings to give actionable reasoning Both `composition_file_too_large` and `timeline_track_too_dense` previously said "Agents produce better results when large scenes are split into smaller sub-compositions." The audience-flavored framing ("Agents produce better results") doesn't tell a reader (agent or human) WHY smaller is better. Reframe to concrete properties of smaller compositions: easier to read, iterate on, and diff. The fixHint already covers the inspect/revise/validate detail; the message now leads with a tight reason. Per Abhay in #C0ACCNHLG3U: > "an agent reading 'Agents produce better results' sounds weird. We should > give the agent an actual reason why smaller is better for them." Co-Authored-By: Claude Opus 4.7 (1M context) --- packages/core/src/lint/rules/composition.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/core/src/lint/rules/composition.ts b/packages/core/src/lint/rules/composition.ts index 0f02b1a4a..7107702e3 100644 --- a/packages/core/src/lint/rules/composition.ts +++ b/packages/core/src/lint/rules/composition.ts @@ -49,7 +49,7 @@ export const compositionRules: Array<(ctx: LintContext) => HyperframeLintFinding { code: "composition_file_too_large", severity: "warning", - message: `This HTML composition file has ${lineCount} lines. Agents produce better results when large scenes are split into smaller sub-compositions.`, + message: `This HTML composition file has ${lineCount} lines. Smaller sub-compositions are easier to read, iterate on, and diff.`, fixHint: `${splitTarget}, then mount them from the parent with data-composition-src so each file stays small enough to inspect, revise, and validate independently.`, }, ]; @@ -77,7 +77,7 @@ export const compositionRules: Array<(ctx: LintContext) => HyperframeLintFinding findings.push({ code: "timeline_track_too_dense", severity: "warning", - message: `Track ${track} has ${count} timed elements in this HTML file. Agents produce better timelines when dense tracks are split into smaller sub-compositions.`, + message: `Track ${track} has ${count} timed elements in this HTML file. Smaller sub-compositions keep timelines easier to read, iterate on, and diff.`, fixHint: `${splitTarget} and mount them from the parent with data-composition-src so the timeline stays easier to inspect, revise, and validate.`, }); } From a707b6a882eb22a6c4f7ccc7e8a24f9de582f940 Mon Sep 17 00:00:00 2001 From: James Date: Tue, 5 May 2026 18:18:23 -0700 Subject: [PATCH 04/13] fix(cli): pin inverse-alpha invariants, harden encoder stdin MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Extract applyMask helper from postprocess and add 5 unit tests pinning the contract this PR is selling: fg.alpha + bg.alpha === 255 per pixel, RGB triples byte-identical between fg and bg, and bg=null path leaves the bg buffer untouched. Without these, a future postprocess change (mask threshold, premultiplied alpha, gamma) could silently break the inverse-alpha relationship and the existing plumbing tests would all still pass. - Add stdin 'error' listener inside spawnFfmpeg. If either encoder dies mid-render, Node emits an unhandled error on the dead writable on the next .write() and crashes the CLI before waitForExit's reject path can surface the encoder's stderr tail. Doubled encoder count = doubled failure surface, so this is worth pinning down. - Tighten stdio param to a 3-tuple so an accidental 1-element array fails at type-check. - Sharpen backpressure comment: write→true means "highWaterMark not exceeded," not "libuv flushed." Reuse-without-corruption is safe only because session.process is slow enough that libuv drains in between. Addresses review on PR #637. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../src/background-removal/inference.test.ts | 111 +++++++++++++++++- .../cli/src/background-removal/inference.ts | 54 ++++++--- .../cli/src/background-removal/pipeline.ts | 18 ++- 3 files changed, 164 insertions(+), 19 deletions(-) diff --git a/packages/cli/src/background-removal/inference.test.ts b/packages/cli/src/background-removal/inference.test.ts index ffc62cc0b..719bcab4d 100644 --- a/packages/cli/src/background-removal/inference.test.ts +++ b/packages/cli/src/background-removal/inference.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { MEAN, STD } from "./inference.js"; +import { MEAN, STD, applyMask } from "./inference.js"; // Regression: the u2net_human_seg model was trained with ImageNet // normalization. Drifting away from these exact values changes the input @@ -16,3 +16,112 @@ describe("background-removal/inference — rembg u2net_human_seg parity", () => expect(STD).toEqual([0.229, 0.224, 0.225]); }); }); + +// These tests pin the contract that `--background-output` is built on: +// fg.alpha + bg.alpha === 255 per pixel, and the RGB plane is byte-identical +// between fg and bg. A future change to the postprocess loop (different mask +// threshold, premultiplied alpha, gamma-corrected compositing) that breaks +// either invariant should fail here loudly. +describe("background-removal/inference — applyMask invariants", () => { + function makeRgb(pixels: number): Buffer { + // Deterministic but non-trivial RGB so byte equality is meaningful. + const buf = Buffer.allocUnsafe(pixels * 3); + for (let i = 0; i < pixels; i++) { + buf[i * 3] = (i * 7) & 0xff; + buf[i * 3 + 1] = (i * 13 + 31) & 0xff; + buf[i * 3 + 2] = (i * 19 + 61) & 0xff; + } + return buf; + } + + function makeMask(pixels: number): Buffer { + // Hit the saturation endpoints (0, 255) and a few mid-tone values so the + // 255-m inversion is exercised across the full byte range. + const buf = Buffer.allocUnsafe(pixels); + for (let i = 0; i < pixels; i++) buf[i] = (i * 37) & 0xff; + return buf; + } + + it("dual-output: fg.alpha + bg.alpha === 255 for every pixel", () => { + const pixels = 64; + const rgb = makeRgb(pixels); + const mask = makeMask(pixels); + const fg = Buffer.allocUnsafe(pixels * 4); + const bg = Buffer.allocUnsafe(pixels * 4); + + const result = applyMask(rgb, mask, fg, bg, pixels); + + expect(result.fg).toBe(fg); + expect(result.bg).toBe(bg); + for (let i = 0; i < pixels; i++) { + const sum = fg[i * 4 + 3]! + bg[i * 4 + 3]!; + expect(sum).toBe(255); + } + }); + + it("dual-output: RGB triples are byte-identical between fg and bg", () => { + const pixels = 64; + const rgb = makeRgb(pixels); + const mask = makeMask(pixels); + const fg = Buffer.allocUnsafe(pixels * 4); + const bg = Buffer.allocUnsafe(pixels * 4); + + applyMask(rgb, mask, fg, bg, pixels); + + for (let i = 0; i < pixels; i++) { + expect(fg[i * 4]).toBe(bg[i * 4]); + expect(fg[i * 4 + 1]).toBe(bg[i * 4 + 1]); + expect(fg[i * 4 + 2]).toBe(bg[i * 4 + 2]); + // And both match the source. + expect(fg[i * 4]).toBe(rgb[i * 3]); + expect(fg[i * 4 + 1]).toBe(rgb[i * 3 + 1]); + expect(fg[i * 4 + 2]).toBe(rgb[i * 3 + 2]); + } + }); + + it("dual-output: fg.alpha equals the input mask", () => { + const pixels = 32; + const rgb = makeRgb(pixels); + const mask = makeMask(pixels); + const fg = Buffer.allocUnsafe(pixels * 4); + const bg = Buffer.allocUnsafe(pixels * 4); + + applyMask(rgb, mask, fg, bg, pixels); + + for (let i = 0; i < pixels; i++) { + expect(fg[i * 4 + 3]).toBe(mask[i]); + } + }); + + it("single-output: bg=null returns bg=null and writes only fg", () => { + const pixels = 32; + const rgb = makeRgb(pixels); + const mask = makeMask(pixels); + const fg = Buffer.allocUnsafe(pixels * 4); + + const result = applyMask(rgb, mask, fg, null, pixels); + + expect(result.bg).toBeNull(); + expect(result.fg).toBe(fg); + for (let i = 0; i < pixels; i++) { + expect(fg[i * 4]).toBe(rgb[i * 3]); + expect(fg[i * 4 + 3]).toBe(mask[i]); + } + }); + + it("saturates correctly at mask=0 and mask=255", () => { + // mask=0 → fg.alpha=0 (transparent subject), bg.alpha=255 (fully opaque plate) + // mask=255 → fg.alpha=255 (fully opaque subject), bg.alpha=0 (transparent plate) + const rgb = Buffer.from([10, 20, 30, 40, 50, 60]); + const mask = Buffer.from([0, 255]); + const fg = Buffer.allocUnsafe(8); + const bg = Buffer.allocUnsafe(8); + + applyMask(rgb, mask, fg, bg, 2); + + expect(fg[3]).toBe(0); + expect(bg[3]).toBe(255); + expect(fg[7]).toBe(255); + expect(bg[7]).toBe(0); + }); +}); diff --git a/packages/cli/src/background-removal/inference.ts b/packages/cli/src/background-removal/inference.ts index 614038663..605257fc3 100644 --- a/packages/cli/src/background-removal/inference.ts +++ b/packages/cli/src/background-removal/inference.ts @@ -200,30 +200,50 @@ async function postprocess( .raw() .toBuffer(); - const pixels = width * height; - if (rgbaBgBuf) { + return applyMask(rgb, fullMask, rgbaBuf, rgbaBgBuf, width * height); +} + +/** + * Composite the RGB source frame with the segmentation mask into one or two + * RGBA buffers. The contract this PR is built on: + * - `fg`'s alpha is the mask, `bg`'s alpha (when provided) is `255 − mask`, + * so `fg.alpha + bg.alpha === 255` for every pixel. + * - RGB triples are byte-identical between `fg` and `bg`. + * - When `bg` is null, only `fg` is touched. + * + * Exported for direct unit testing of the invariants above without spinning + * up an ONNX session. + */ +export function applyMask( + rgb: Buffer, + mask: Buffer, + fg: Buffer, + bg: Buffer | null, + pixels: number, +): SessionResult { + if (bg) { for (let i = 0; i < pixels; i++) { const r = rgb[i * 3]!; const g = rgb[i * 3 + 1]!; const b = rgb[i * 3 + 2]!; - const m = fullMask[i]!; + const m = mask[i]!; const o = i * 4; - rgbaBuf[o] = r; - rgbaBuf[o + 1] = g; - rgbaBuf[o + 2] = b; - rgbaBuf[o + 3] = m; - rgbaBgBuf[o] = r; - rgbaBgBuf[o + 1] = g; - rgbaBgBuf[o + 2] = b; - rgbaBgBuf[o + 3] = 255 - m; + fg[o] = r; + fg[o + 1] = g; + fg[o + 2] = b; + fg[o + 3] = m; + bg[o] = r; + bg[o + 1] = g; + bg[o + 2] = b; + bg[o + 3] = 255 - m; } - return { fg: rgbaBuf, bg: rgbaBgBuf }; + return { fg, bg }; } for (let i = 0; i < pixels; i++) { - rgbaBuf[i * 4] = rgb[i * 3]!; - rgbaBuf[i * 4 + 1] = rgb[i * 3 + 1]!; - rgbaBuf[i * 4 + 2] = rgb[i * 3 + 2]!; - rgbaBuf[i * 4 + 3] = fullMask[i]!; + fg[i * 4] = rgb[i * 3]!; + fg[i * 4 + 1] = rgb[i * 3 + 1]!; + fg[i * 4 + 2] = rgb[i * 3 + 2]!; + fg[i * 4 + 3] = mask[i]!; } - return { fg: rgbaBuf, bg: null }; + return { fg, bg: null }; } diff --git a/packages/cli/src/background-removal/pipeline.ts b/packages/cli/src/background-removal/pipeline.ts index cbd60a798..a0a0f3bfc 100644 --- a/packages/cli/src/background-removal/pipeline.ts +++ b/packages/cli/src/background-removal/pipeline.ts @@ -318,12 +318,21 @@ interface FfmpegProc { getStderr: () => string; } -function spawnFfmpeg(args: string[], label: string, stdio: ("ignore" | "pipe")[]): FfmpegProc { +type StdioFd = "ignore" | "pipe"; +type StdioTuple = [StdioFd, StdioFd, StdioFd]; + +function spawnFfmpeg(args: string[], label: string, stdio: StdioTuple): FfmpegProc { const proc = spawn("ffmpeg", args, { stdio }); let stderrBuf = ""; proc.stderr?.on("data", (d: Buffer) => { stderrBuf += d.toString(); }); + // If the encoder dies mid-render, the next .write() to its stdin emits an + // 'error' event on the writable. Without a listener, Node treats it as + // unhandled and crashes the CLI before waitForExit's reject path can + // surface the real cause (encoder stderr tail). Swallowing here is safe — + // the process exit is the source of truth. + proc.stdin?.on("error", () => {}); const exit = waitForExit(proc, label, () => stderrBuf); return { proc, exit, getStderr: () => stderrBuf }; } @@ -383,6 +392,13 @@ async function runPipeline( // Issue both writes before any await so a slow encoder doesn't block // the other. Drain anything that returned false before the next // session.process() — its output buffers are reused per frame. + // + // Subtlety: write() returning true means "highWaterMark not exceeded," + // NOT "libuv has flushed the chunk." The buffer reference is held by + // libuv until the underlying syscall completes. Reusing the session's + // output buffer is safe because the next session.process() call takes + // ~10–50ms (ORT inference) — plenty of event-loop turns for libuv to + // drain. If that ever stops being true, we'd need to copy here. const fgWroteFully = fg.proc.stdin!.write(result.fg); const bgWroteFully = bg && result.bg ? bg.proc.stdin!.write(result.bg) : true; if (!fgWroteFully || !bgWroteFully) { From af2f727b3f8c7cc074c539460ac34372f1d8a064 Mon Sep 17 00:00:00 2001 From: Rames Jusso Date: Wed, 6 May 2026 02:43:58 +0000 Subject: [PATCH 05/13] fix(bundler): inline runtime body, drop bare-semi joins, drop empty catch binding MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three issues in `bundleToSingleHtml` reported via Abhay's LLM-based code-validity eval against the bundled output. Each is independently small; they share a single PR because they're all artifacts of the bundler-output shape. 1. Empty `src=""` runtime placeholder (real bug) `htmlBundler.ts:injectInterceptor` emitted `` when no `HYPERFRAME_RUNTIME_URL` was configured. Empty `src` resolves to the page URL itself; Chrome flags this as an infinite-fetch hazard. Three other consumers (studioServer, validate, snapshot) post-process the placeholder to substitute either a real URL or an inlined body — `bundleToSingleHtml` did not, so the bundle wasn't actually self-contained despite the function name. Fix: when no URL is configured, inline the runtime IIFE directly via `getHyperframeRuntimeScript()`. Otherwise emit `src=…` as before. 2. Bare-semicolon lines between joined JS chunks (cosmetic) Three sites used `chunks.join("\n;\n")` (body-script coalesce, local JS, composition scripts) which produced a lone `;` on its own line between chunks. Valid JS but a code smell. Replace with a `joinJsChunks()` helper that ensures each chunk ends in `;` and joins on `\n`. 3. Empty `catch (_err) {}` in compositionScoping.ts (lint-noisy) The `_err` underscore prefix signals "intentionally swallowed" but bundle-time linters often don't honor that convention. Replaced with `catch { /* ... */ }` (no binding, explanatory comment) — same behavior, no rule fires. Tests: 2 new regression guards (runtime-not-empty-src, no-bare-semi) plus existing tests updated to reflect the new inlined-runtime shape (the previous "runtime block must not contain getElementById" assertion no longer holds because the inlined body itself uses getElementById; replaced with a more specific "author script not merged into runtime tag" check). Issue #4 from the original report (Unterminated string at line 1111 col 18, char 65497) was not directly reproducible after applying these fixes — esbuild parses all 4 inline scripts in the rebundled output cleanly. The unterminated- string symptom was likely a downstream artifact of the bare-semicolon joining or the empty-src placeholder confusing the lint tool. If the original symptom persists on a clean re-run against the fixed bundle, will open a follow-up PR with a focused repro. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../core/src/compiler/compositionScoping.ts | 6 +- .../core/src/compiler/htmlBundler.test.ts | 84 ++++++++++++++++++- packages/core/src/compiler/htmlBundler.ts | 44 +++++++--- 3 files changed, 120 insertions(+), 14 deletions(-) diff --git a/packages/core/src/compiler/compositionScoping.ts b/packages/core/src/compiler/compositionScoping.ts index ea3c55953..5a5030e16 100644 --- a/packages/core/src/compiler/compositionScoping.ts +++ b/packages/core/src/compiler/compositionScoping.ts @@ -212,7 +212,11 @@ export function wrapScopedCompositionScript( value: __hfFindRoot(), configurable: true, }); - } catch (_err) {} + } catch { + // Best-effort: timelines coming from user code may have a frozen target + // or a non-extensible defineProperty path. Swallow — the scoped root + // is an enrichment, not a correctness invariant for playback. + } return timeline; }; var __hfBaseGsap = typeof gsap === "undefined" ? window.gsap : gsap; diff --git a/packages/core/src/compiler/htmlBundler.test.ts b/packages/core/src/compiler/htmlBundler.test.ts index e6a24ddb0..d9686b849 100644 --- a/packages/core/src/compiler/htmlBundler.test.ts +++ b/packages/core/src/compiler/htmlBundler.test.ts @@ -38,10 +38,82 @@ describe("bundleToSingleHtml", () => { )?.[0]; expect(runtimeBlock).toBeDefined(); - expect(runtimeBlock).not.toContain("getElementById"); + // The runtime block must contain the inlined HF runtime IIFE — bundled + // output is self-contained, so the bundle's runtime body is loaded inline, + // not referenced via src. + expect(runtimeBlock).toMatch(/data-hyperframes-preview-runtime="1">/); + expect(runtimeBlock).not.toMatch(/src=""/); + // The author's specific composition script must NOT be merged INTO the + // runtime tag — it stays as its own when no runtime URL was configured. An + // empty src resolves to the page URL itself, which Chrome flags as an + // infinite-fetch hazard. Verify that bundleToSingleHtml inlines the + // runtime body so the bundle is genuinely self-contained. + const dir = makeTempProject({ + "index.html": ` + +
+`, + }); + + const previousUrl = process.env.HYPERFRAME_RUNTIME_URL; + delete process.env.HYPERFRAME_RUNTIME_URL; + let bundled: string; + try { + bundled = await bundleToSingleHtml(dir); + } finally { + if (previousUrl !== undefined) process.env.HYPERFRAME_RUNTIME_URL = previousUrl; + } + + const runtimeBlock = bundled.match( + /]*data-hyperframes-preview-runtime[^>]*>[\s\S]*?<\/script>/i, + )?.[0]; + expect(runtimeBlock).toBeDefined(); + // Must NOT have an empty src attribute (would self-fetch). + expect(runtimeBlock).not.toMatch(/src=""/); + // Must have a non-trivial inlined body (the runtime IIFE is ~150KB). + const innerLength = (runtimeBlock!.match(/>([\s\S]*?)<\/script>/)?.[1] ?? "").length; + expect(innerLength).toBeGreaterThan(1000); + }); + + it("does not produce stray bare-semicolon lines between concatenated JS chunks", async () => { + // Regression guard: hf#XXX. Earlier the bundler joined script chunks with + // `\n;\n`, which produces a lone `;` on its own line between chunks. Valid + // JS but reads as a code smell. Each chunk should end in `;` and chunks + // should join with `\n`. + const dir = makeTempProject({ + "index.html": ` + +
+
+
+ + + +`, + "local-a.js": "window.__a = 1", + "local-b.js": "window.__b = 2", + "compositions/child.html": ``, + }); + + const bundled = await bundleToSingleHtml(dir); + // No line is JUST a bare semicolon (with optional surrounding whitespace). + expect(bundled).not.toMatch(/\n\s*;\s*\n/); + }); + it("hoists external CDN scripts from sub-compositions into the bundle", async () => { const dir = makeTempProject({ "index.html": ` @@ -84,8 +156,14 @@ describe("bundleToSingleHtml", () => { // GSAP CDN from main doc should still be present expect(bundled).toContain("cdn.jsdelivr.net/npm/gsap"); - // data-composition-src should be stripped (composition was inlined) - expect(bundled).not.toContain("data-composition-src"); + // data-composition-src should be stripped from the host element (composition + // was inlined). The literal string may still appear inside the inlined + // runtime IIFE that knows how to look up that attribute — so check the DOM, + // not the raw text. + const { document: doc } = parseHTML(bundled); + const hostEl = doc.getElementById("rockets-host"); + expect(hostEl).toBeTruthy(); + expect(hostEl?.hasAttribute("data-composition-src")).toBe(false); }); it("does not duplicate CDN scripts already present in the main document", async () => { diff --git a/packages/core/src/compiler/htmlBundler.ts b/packages/core/src/compiler/htmlBundler.ts index 3a80d58c9..00cae2a0e 100644 --- a/packages/core/src/compiler/htmlBundler.ts +++ b/packages/core/src/compiler/htmlBundler.ts @@ -10,6 +10,7 @@ import { import { rewriteAssetPaths, rewriteCssAssetUrls } from "./rewriteSubCompPaths"; import { scopeCssToComposition, wrapScopedCompositionScript } from "./compositionScoping"; import { validateHyperframeHtmlContract } from "./staticGuard"; +import { getHyperframeRuntimeScript } from "../generated/runtime-inline"; /** Resolve a relative path within projectDir, rejecting traversal outside it. */ function safePath(projectDir: string, relativePath: string): string | null { @@ -30,8 +31,20 @@ function injectInterceptor(html: string): string { const sanitized = stripEmbeddedRuntimeScripts(html); if (sanitized.includes(RUNTIME_BOOTSTRAP_ATTR)) return sanitized; - const runtimeScriptUrl = getRuntimeScriptUrl().replace(/"/g, """); - const tag = ``; + // When a runtime URL is configured (HYPERFRAME_RUNTIME_URL env var), the bundle + // points at it via src=… and the host page serves the script. When no URL is + // configured — the common `bundleToSingleHtml` use case — inline the runtime + // body so the bundle is genuinely self-contained. An empty src="" attribute + // would otherwise resolve to the page URL and trigger an infinite-fetch loop. + const runtimeScriptUrl = getRuntimeScriptUrl(); + let tag: string; + if (runtimeScriptUrl) { + const escaped = runtimeScriptUrl.replace(/"/g, """); + tag = ``; + } else { + const inlinedRuntime = getHyperframeRuntimeScript(); + tag = ``; + } if (sanitized.includes("")) { return sanitized.replace("", `${tag}\n`); } @@ -268,11 +281,7 @@ function coalesceHeadStylesAndBodyScripts(document: Document): void { return !type || type === "text/javascript" || type === "application/javascript"; }); if (bodyInlineScripts.length > 0) { - const mergedJs = bodyInlineScripts - .map((el) => (el.textContent || "").trim()) - .filter(Boolean) - .join("\n;\n") - .trim(); + const mergedJs = joinJsChunks(bodyInlineScripts.map((el) => el.textContent || "")); for (const el of bodyInlineScripts) el.remove(); if (mergedJs) { const stripped = stripJsCommentsParserSafe(mergedJs); @@ -283,6 +292,20 @@ function coalesceHeadStylesAndBodyScripts(document: Document): void { } } +/** + * Concatenate JS chunks safely. Each chunk gets a trailing `;` if it doesn't + * already end in one, so the joined output never inserts a stray bare-semicolon + * line between chunks (the `\n;\n` separator pattern produces a lone `;` on its + * own line, which is valid JS but reads as a code smell to most linters). + */ +function joinJsChunks(chunks: string[]): string { + return chunks + .map((chunk) => chunk.trim()) + .filter((chunk) => chunk.length > 0) + .map((chunk) => (chunk.endsWith(";") ? chunk : chunk + ";")) + .join("\n"); +} + function stripJsCommentsParserSafe(source: string): string { if (!source) return source; try { @@ -379,12 +402,13 @@ export async function bundleToSingleHtml( } if (localJsChunks.length > 0) { const anchor = document.querySelector('script[data-hf-bundled-local-js="1"]'); + const joinedJs = joinJsChunks(localJsChunks); if (anchor) { anchor.removeAttribute("data-hf-bundled-local-js"); - anchor.textContent = localJsChunks.join("\n;\n"); + anchor.textContent = joinedJs; } else { const script = document.createElement("script"); - script.textContent = localJsChunks.join("\n;\n"); + script.textContent = joinedJs; document.body.appendChild(script); } } @@ -623,7 +647,7 @@ export async function bundleToSingleHtml( } if (compScriptChunks.length) { const compScript = document.createElement("script"); - compScript.textContent = compScriptChunks.join("\n;\n"); + compScript.textContent = joinJsChunks(compScriptChunks); document.body.appendChild(compScript); } From dfca302d3747b6d1abd919c1d1cdd879fe92b7d8 Mon Sep 17 00:00:00 2001 From: Rames Jusso Date: Wed, 6 May 2026 04:37:04 +0000 Subject: [PATCH 06/13] fix(bundler): runtime mode opt-in, ASI-safe joinJsChunks, prune dead subs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per @vai-bot's review on hf#641: Important #1: dead `src=""` substitution sites ============================================= Now that `bundleToSingleHtml` inlines the runtime IIFE by default, the empty `src=""` placeholder is never emitted in the no-env-var path — the 5 downstream substitution sites that grep for `src=""` were dead. Two of them (studio dev server + studio vite preview) genuinely WANT the placeholder so they can hot-reload a local /api/runtime.js endpoint without re-inlining ~150 KB on every composition edit. Three of them (CLI validate, snapshot, layout) were just doing the same inlining the bundler already does. Resolution: - Add a `runtime: "inline" | "placeholder"` option to `BundleOptions`. Default is "inline" (matches the self-contained-bundle promise the function name makes). The two studio surfaces explicitly pass `{ runtime: "placeholder" }` to opt in. - studioServer.ts + studio/vite.config.ts: pass the option, keep their existing string-replace logic unchanged. - validate.ts + snapshot.ts + layout.ts: delete the now-redundant runtime substitution code (regex never matches the new inlined-runtime shape). Important #2: joinJsChunks ASI hazard ====================================== The new helper appended `;` to chunks not already ending in `;` and joined on `\n`. If a chunk ended with a `// line comment`, the appended semicolon was eaten by the comment, leaving the next chunk's first statement attached to the previous chunk's last expression — exactly the ASI hazard the helper exists to prevent. Fix: append `\n;` instead of `;` for chunks not already terminated. The newline closes the line comment, the standalone `;` becomes the statement separator. For typical chunks (already ending in `;`), output is unchanged — still clean `\n`-joined chunks with no bare-semicolon lines. Also added a trailing `;` to `wrapScopedCompositionScript`'s IIFE close (`})()` → `})();`) so composition scripts join cleanly without falling through to the `\n;` fallback. New test: regression guard at the chunk boundary verifies every inline script body in the bundle parses cleanly via esbuild even when a source JS file ends with a line comment. Verification ============ - `bun run --filter @hyperframes/core test` — 653/653 pass - `bun run --filter @hyperframes/cli test` — 243/243 pass - `bun run --filter @hyperframes/{core,cli,studio} typecheck` — clean - `bunx oxfmt --check` + `bunx oxlint` on all touched files — clean Co-Authored-By: Claude Opus 4.7 (1M context) --- packages/cli/src/commands/layout.ts | 25 ++------- packages/cli/src/commands/snapshot.ts | 17 +----- packages/cli/src/commands/validate.ts | 24 ++------ packages/cli/src/server/studioServer.ts | 7 ++- .../core/src/compiler/compositionScoping.ts | 2 +- .../core/src/compiler/htmlBundler.test.ts | 34 ++++++++++++ packages/core/src/compiler/htmlBundler.ts | 55 +++++++++++++++---- packages/studio/vite.config.ts | 12 ++-- 8 files changed, 103 insertions(+), 73 deletions(-) diff --git a/packages/cli/src/commands/layout.ts b/packages/cli/src/commands/layout.ts index a8a713963..fa27dd70b 100644 --- a/packages/cli/src/commands/layout.ts +++ b/packages/cli/src/commands/layout.ts @@ -1,6 +1,6 @@ import { defineCommand } from "citty"; import { existsSync, readFileSync } from "node:fs"; -import { dirname, join, resolve } from "node:path"; +import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; import type { Example } from "./_examples.js"; import { c } from "../ui/colors.js"; @@ -107,27 +107,10 @@ async function seekTo(page: import("puppeteer-core").Page, time: number): Promis } async function bundleProjectHtml(projectDir: string): Promise { + // `bundleToSingleHtml` now inlines the runtime IIFE by default, so the + // previous post-bundle runtime substitution is no longer needed. const { bundleToSingleHtml } = await import("@hyperframes/core/compiler"); - let html = await bundleToSingleHtml(projectDir); - - const runtimePath = resolve( - __dirname, - "..", - "..", - "..", - "core", - "dist", - "hyperframe.runtime.iife.js", - ); - if (existsSync(runtimePath)) { - const runtimeSource = readFileSync(runtimePath, "utf-8"); - html = html.replace( - /]*data-hyperframes-preview-runtime[^>]*src="[^"]*"[^>]*><\/script>/, - () => ``, - ); - } - - return html; + return bundleToSingleHtml(projectDir); } async function alignViewportToComposition( diff --git a/packages/cli/src/commands/snapshot.ts b/packages/cli/src/commands/snapshot.ts index 2c1d26cce..0fd2a81da 100644 --- a/packages/cli/src/commands/snapshot.ts +++ b/packages/cli/src/commands/snapshot.ts @@ -97,20 +97,9 @@ async function captureSnapshots( const numFrames = opts.frames ?? 5; - // 1. Bundle - let html = await bundleToSingleHtml(projectDir); - - // Inject local runtime if available. - // Uses the same multi-strategy resolver as the studio preview server - // (runtimeSource.ts) so snapshot works in dev (tsx), built CLI, and npx. - const { loadRuntimeSource } = await import("../server/runtimeSource.js"); - const runtimeSource = await loadRuntimeSource(); - if (runtimeSource) { - html = html.replace( - /]*data-hyperframes-preview-runtime[^>]*src="[^"]*"[^>]*><\/script>/, - () => ``, - ); - } + // 1. Bundle. `bundleToSingleHtml` now inlines the runtime IIFE by default, + // so the previous post-bundle runtime substitution is no longer needed. + const html = await bundleToSingleHtml(projectDir); const server = await serveStaticProjectHtml(projectDir, html); diff --git a/packages/cli/src/commands/validate.ts b/packages/cli/src/commands/validate.ts index ed2bd7f99..8c9a2fba0 100644 --- a/packages/cli/src/commands/validate.ts +++ b/packages/cli/src/commands/validate.ts @@ -1,6 +1,6 @@ import { defineCommand } from "citty"; import { existsSync, readFileSync } from "node:fs"; -import { resolve, join, dirname } from "node:path"; +import { join, dirname } from "node:path"; import { fileURLToPath } from "node:url"; import { resolveProject } from "../utils/project.js"; import { resolveCompositionViewportFromHtml } from "../utils/compositionViewport.js"; @@ -113,24 +113,10 @@ async function validateInBrowser( const { bundleToSingleHtml } = await import("@hyperframes/core/compiler"); const { ensureBrowser } = await import("../browser/manager.js"); - let html = await bundleToSingleHtml(projectDir); - - const runtimePath = resolve( - __dirname, - "..", - "..", - "..", - "core", - "dist", - "hyperframe.runtime.iife.js", - ); - if (existsSync(runtimePath)) { - const runtimeSource = readFileSync(runtimePath, "utf-8"); - html = html.replace( - /]*data-hyperframes-preview-runtime[^>]*src="[^"]*"[^>]*><\/script>/, - () => ``, - ); - } + // `bundleToSingleHtml` now inlines the runtime IIFE by default, so the + // previous post-bundle regex substitution (which matched `src="..."` on the + // runtime tag) is no longer needed — there's no `src` attribute to match. + const html = await bundleToSingleHtml(projectDir); const { createServer } = await import("node:http"); const { getMimeType } = await import("@hyperframes/core/studio-api"); diff --git a/packages/cli/src/server/studioServer.ts b/packages/cli/src/server/studioServer.ts index ccceccffb..3397d7f33 100644 --- a/packages/cli/src/server/studioServer.ts +++ b/packages/cli/src/server/studioServer.ts @@ -153,8 +153,11 @@ export function createStudioServer(options: StudioServerOptions): StudioServer { async bundle(dir: string): Promise { try { const { bundleToSingleHtml } = await import("@hyperframes/core/compiler"); - let html = await bundleToSingleHtml(dir); - // Fix empty runtime src from bundler — point to the local runtime endpoint + // Studio dev server: ask the bundler for an empty `src=""` placeholder so + // we can point it at our hot-reloadable local runtime endpoint. Inlining + // ~150 KB of runtime body on every preview render would defeat browser + // caching across composition edits. + let html = await bundleToSingleHtml(dir, { runtime: "placeholder" }); html = html.replace( 'data-hyperframes-preview-runtime="1" src=""', 'data-hyperframes-preview-runtime="1" src="/api/runtime.js"', diff --git a/packages/core/src/compiler/compositionScoping.ts b/packages/core/src/compiler/compositionScoping.ts index 5a5030e16..bc7153011 100644 --- a/packages/core/src/compiler/compositionScoping.ts +++ b/packages/core/src/compiler/compositionScoping.ts @@ -286,5 +286,5 @@ ${source} }; __hfFindRoot(); __hfRun(); -})()`; +})();`; } diff --git a/packages/core/src/compiler/htmlBundler.test.ts b/packages/core/src/compiler/htmlBundler.test.ts index d9686b849..225072391 100644 --- a/packages/core/src/compiler/htmlBundler.test.ts +++ b/packages/core/src/compiler/htmlBundler.test.ts @@ -82,6 +82,40 @@ describe("bundleToSingleHtml", () => { expect(innerLength).toBeGreaterThan(1000); }); + it("preserves chunk integrity when a chunk ends with a line comment (ASI hazard guard)", async () => { + // Regression guard for the joinJsChunks helper. If a chunk ends with `// ...` + // and we naively appended `;` on the same line, the appended semicolon would + // be eaten by the comment, leaving the next chunk's first statement attached + // to the previous chunk's last expression. Verify the helper appends `\n;` + // instead so the comment terminates and the semicolon stands alone. + const dir = makeTempProject({ + "index.html": ` + +
+ + + +`, + // Chunk A ends with a // line comment — without the \n separator before + // the appended ;, that ; would be eaten by the comment. + "local-a.js": "window.__a = 1 // trailing line comment", + "local-b.js": "window.__b = 2", + }); + + const bundled = await bundleToSingleHtml(dir); + // Run every inline script body through esbuild; if the line comment ate + // the separator, parse would fail with an unexpected-token error somewhere + // around the chunk boundary. + const { transformSync } = await import("esbuild"); + const re = /]*>([\s\S]*?)<\/script>/g; + let m: RegExpExecArray | null; + while ((m = re.exec(bundled)) !== null) { + const body = m[1]; + if (!body || !body.trim()) continue; + expect(() => transformSync(body, { loader: "js", minify: false })).not.toThrow(); + } + }); + it("does not produce stray bare-semicolon lines between concatenated JS chunks", async () => { // Regression guard: hf#XXX. Earlier the bundler joined script chunks with // `\n;\n`, which produces a lone `;` on its own line between chunks. Valid diff --git a/packages/core/src/compiler/htmlBundler.ts b/packages/core/src/compiler/htmlBundler.ts index 00cae2a0e..def7c8463 100644 --- a/packages/core/src/compiler/htmlBundler.ts +++ b/packages/core/src/compiler/htmlBundler.ts @@ -27,20 +27,24 @@ function getRuntimeScriptUrl(): string { return configured || DEFAULT_RUNTIME_SCRIPT_URL; } -function injectInterceptor(html: string): string { +function injectInterceptor(html: string, runtimeMode: "inline" | "placeholder" = "inline"): string { const sanitized = stripEmbeddedRuntimeScripts(html); if (sanitized.includes(RUNTIME_BOOTSTRAP_ATTR)) return sanitized; - // When a runtime URL is configured (HYPERFRAME_RUNTIME_URL env var), the bundle - // points at it via src=… and the host page serves the script. When no URL is - // configured — the common `bundleToSingleHtml` use case — inline the runtime - // body so the bundle is genuinely self-contained. An empty src="" attribute - // would otherwise resolve to the page URL and trigger an infinite-fetch loop. + // Three modes for the runtime `; + } else if (runtimeMode === "placeholder") { + tag = ``; } else { const inlinedRuntime = getHyperframeRuntimeScript(); tag = ``; @@ -293,16 +297,27 @@ function coalesceHeadStylesAndBodyScripts(document: Document): void { } /** - * Concatenate JS chunks safely. Each chunk gets a trailing `;` if it doesn't - * already end in one, so the joined output never inserts a stray bare-semicolon - * line between chunks (the `\n;\n` separator pattern produces a lone `;` on its - * own line, which is valid JS but reads as a code smell to most linters). + * Concatenate JS chunks safely. Goals: + * - Each chunk's last statement is terminated, so joining can't introduce ASI + * surprises (e.g. `a()` followed by `(b)()` — the second chunk would parse + * as a call on the first's return value). + * - In the common case (chunk already ends with `;` — typical of esbuild + * output and IIFE-wrapped composition scripts ending in `})();`), the join + * produces clean output: chunks separated by `\n` with no stray bare + * semicolon lines. + * - Defensive against trailing line comments. If a chunk ends with `// ...` + * and we appended `;` on the same line, the appended semicolon would be + * swallowed by the comment, leaving the next chunk's first statement + * attached to the previous chunk's last expression — exactly the ASI + * hazard this helper exists to prevent. So when a chunk doesn't already + * end in `;`, we append `\n;` instead — the newline closes any line + * comment, and the standalone `;` becomes the statement separator. */ function joinJsChunks(chunks: string[]): string { return chunks .map((chunk) => chunk.trim()) .filter((chunk) => chunk.length > 0) - .map((chunk) => (chunk.endsWith(";") ? chunk : chunk + ";")) + .map((chunk) => (chunk.endsWith(";") ? chunk : chunk + "\n;")) .join("\n"); } @@ -319,6 +334,22 @@ function stripJsCommentsParserSafe(source: string): string { export interface BundleOptions { /** Optional media duration prober (e.g., ffprobe). If omitted, media durations are not resolved. */ probeMediaDuration?: MediaDurationProber; + /** + * How to handle the HyperFrames runtime ` so the caller can + * substitute it with a real URL via string replace. Used by the dev studio + * server and vite preview to point at a local runtime endpoint, which keeps + * the runtime cacheable across hot-reloads instead of re-inlining ~150 KB + * on every change. + * + * The `HYPERFRAME_RUNTIME_URL` env var, when set, takes precedence over both + * modes and emits `` regex as case-sensitive, which would miss `` as too strict — `` (with whitespace before `>`) is valid HTML and would slip past the matcher. Changed to `` for full defense-in-depth. The bundler always emits the canonical form, so no real-traffic miss — this is hardening the test's parse-loop, not fixing a downstream bug. Addresses CodeQL alert on #641. --- packages/core/src/compiler/htmlBundler.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/core/src/compiler/htmlBundler.test.ts b/packages/core/src/compiler/htmlBundler.test.ts index aa2ccbc7b..7675b307f 100644 --- a/packages/core/src/compiler/htmlBundler.test.ts +++ b/packages/core/src/compiler/htmlBundler.test.ts @@ -107,7 +107,7 @@ describe("bundleToSingleHtml", () => { // the separator, parse would fail with an unexpected-token error somewhere // around the chunk boundary. const { transformSync } = await import("esbuild"); - const re = /]*>([\s\S]*?)<\/script>/gi; + const re = /]*>([\s\S]*?)<\/script\s*>/gi; let m: RegExpExecArray | null; while ((m = re.exec(bundled)) !== null) { const body = m[1]; From f3f542b42f48fa88229b245332ecaa55c349e1b0 Mon Sep 17 00:00:00 2001 From: James Date: Wed, 6 May 2026 05:04:43 +0000 Subject: [PATCH 09/13] test(bundler): accept arbitrary content in script close tag MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CodeQL still flagged `` as too narrow — the rule wants tolerance for `` (HTML parser treats trailing content in a close tag as part of the tag). Switched to `]*>` for full coverage. The bundler still always emits the canonical ``; this is test-side hardening, not a runtime fix. --- packages/core/src/compiler/htmlBundler.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/core/src/compiler/htmlBundler.test.ts b/packages/core/src/compiler/htmlBundler.test.ts index 7675b307f..c70f49037 100644 --- a/packages/core/src/compiler/htmlBundler.test.ts +++ b/packages/core/src/compiler/htmlBundler.test.ts @@ -107,7 +107,7 @@ describe("bundleToSingleHtml", () => { // the separator, parse would fail with an unexpected-token error somewhere // around the chunk boundary. const { transformSync } = await import("esbuild"); - const re = /]*>([\s\S]*?)<\/script\s*>/gi; + const re = /]*>([\s\S]*?)<\/script[^>]*>/gi; let m: RegExpExecArray | null; while ((m = re.exec(bundled)) !== null) { const body = m[1]; From 3d370c40640ea1b5546d01997e3676a49984c489 Mon Sep 17 00:00:00 2001 From: James Date: Wed, 6 May 2026 05:06:42 +0000 Subject: [PATCH 10/13] test(bundler): parse scripts via linkedom, not regex MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per CodeQL's `js/bad-tag-filter` recommendation, replace the regex-based `', + ); + + const srcdoc = player.iframeElement.srcdoc; + expect(srcdoc).toContain('window.__HF_SHADER_CAPTURE_SCALE="0.5";'); + expect(srcdoc).toContain('window.__HF_SHADER_LOADING="player";'); + expect(srcdoc.indexOf("data-hyperframes-player-shader-options")).toBeLessThan( + srcdoc.indexOf("composition.js"), + ); + }); + + it("shows and hides the player-owned shader loader from transition state messages", () => { + vi.useFakeTimers(); + const player = document.createElement("hyperframes-player") as PlayerWithIframe; + player.setAttribute("shader-loading", "player"); + document.body.appendChild(player); + + const iframeWindow = player.iframeElement.contentWindow; + expect(iframeWindow).toBeTruthy(); + window.dispatchEvent( + new MessageEvent("message", { + source: iframeWindow, + data: { + source: "hf-preview", + type: "shader-transition-state", + compositionId: "main", + state: { + loading: true, + progress: 3, + total: 10, + currentTransition: 1, + transitionTotal: 2, + transitionFrame: 3, + transitionFrames: 5, + phase: "capturing", + }, + }, + }), + ); + + const loader = player.shadowRoot?.querySelector(".hfp-shader-loader"); + expect(loader?.classList.contains("hfp-visible")).toBe(true); + expect(loader?.textContent).toContain("1/2"); + expect(loader?.textContent).toContain("3/5"); + + const playEvents: Event[] = []; + player.addEventListener("play", (event) => playEvents.push(event)); + loader?.dispatchEvent(new MouseEvent("click", { bubbles: true, composed: true })); + expect(playEvents).toHaveLength(0); + + window.dispatchEvent( + new MessageEvent("message", { + source: iframeWindow, + data: { + source: "hf-preview", + type: "shader-transition-state", + compositionId: "main", + state: { loading: false, ready: true }, + }, + }), + ); + window.dispatchEvent( + new MessageEvent("message", { + source: iframeWindow, + data: { + source: "hf-preview", + type: "shader-transition-state", + compositionId: "main", + state: { loading: false, ready: true }, + }, + }), + ); + expect(loader?.classList.contains("hfp-visible")).toBe(false); + expect(loader?.classList.contains("hfp-hiding")).toBe(true); + vi.advanceTimersByTime(420); + expect(loader?.classList.contains("hfp-hiding")).toBe(false); + vi.useRealTimers(); + }); +}); + // ── Shared stylesheet (adoptedStyleSheets) ── // // Every player constructed in the same document should adopt the *same* diff --git a/packages/player/src/hyperframes-player.ts b/packages/player/src/hyperframes-player.ts index 0c180f731..a9cd174d0 100644 --- a/packages/player/src/hyperframes-player.ts +++ b/packages/player/src/hyperframes-player.ts @@ -20,6 +20,114 @@ function getSharedSheet(): CSSStyleSheet | null { const DEFAULT_FPS = 30; const RUNTIME_CDN_URL = "https://cdn.jsdelivr.net/npm/@hyperframes/core/dist/hyperframe.runtime.iife.js"; +const SHADER_CAPTURE_SCALE_ATTR = "shader-capture-scale"; +const SHADER_LOADING_ATTR = "shader-loading"; +const SHADER_CAPTURE_SCALE_PARAM = "__hf_shader_capture_scale"; +const SHADER_LOADING_PARAM = "__hf_shader_loading"; + +export type ShaderLoadingMode = "composition" | "player" | "none"; + +interface ShaderTransitionState { + ready?: boolean; + progress?: number; + total?: number; + currentTransition?: number; + transitionTotal?: number; + transitionFrame?: number; + transitionFrames?: number; + phase?: "cached" | "capturing" | "finalizing"; + loading?: boolean; +} + +interface ShaderLoaderElements { + root: HTMLDivElement; + fill: HTMLDivElement; + title: HTMLSpanElement; + detail: HTMLDivElement; + transitionValue: HTMLSpanElement; + frameLabel: HTMLSpanElement; + frameValue: HTMLSpanElement; + frameRow: HTMLDivElement; +} + +const SHADER_LOADING_PHRASES = [ + "Preparing scene transitions", + "Sampling outgoing scene motion", + "Sampling incoming scene motion", + "Caching transition frames", + "Finalizing transition preview", +]; + +function normalizeShaderCaptureScale(value: string | null): string | null { + if (value === null) return null; + const parsed = Number(value); + if (!Number.isFinite(parsed) || parsed <= 0) return null; + return String(Math.min(1, Math.max(0.25, parsed))); +} + +function normalizeShaderLoadingMode(value: string | null): ShaderLoadingMode { + if (value === null || value.trim() === "") return "composition"; + const normalized = value.trim().toLowerCase(); + if ( + normalized === "none" || + normalized === "false" || + normalized === "0" || + normalized === "off" + ) { + return "none"; + } + if ( + normalized === "player" || + normalized === "true" || + normalized === "1" || + normalized === "on" + ) { + return "player"; + } + return "composition"; +} + +function setQueryParam(params: URLSearchParams, key: string, value: string | null): void { + if (value === null) params.delete(key); + else params.set(key, value); +} + +function withShaderQueryParams( + src: string, + scale: string | null, + loadingMode: ShaderLoadingMode, +): string { + const hashIndex = src.indexOf("#"); + const beforeHash = hashIndex >= 0 ? src.slice(0, hashIndex) : src; + const hash = hashIndex >= 0 ? src.slice(hashIndex) : ""; + const queryIndex = beforeHash.indexOf("?"); + const path = queryIndex >= 0 ? beforeHash.slice(0, queryIndex) : beforeHash; + const query = queryIndex >= 0 ? beforeHash.slice(queryIndex + 1) : ""; + const params = new URLSearchParams(query); + setQueryParam(params, SHADER_CAPTURE_SCALE_PARAM, scale); + setQueryParam(params, SHADER_LOADING_PARAM, loadingMode === "composition" ? null : loadingMode); + const nextQuery = params.toString(); + return `${path}${nextQuery ? `?${nextQuery}` : ""}${hash}`; +} + +function injectShaderOptionsIntoSrcdoc( + html: string, + scale: string | null, + loadingMode: ShaderLoadingMode, +): string { + if (scale === null && loadingMode === "composition") return html; + const lines: string[] = []; + if (scale !== null) lines.push(`window.__HF_SHADER_CAPTURE_SCALE=${JSON.stringify(scale)};`); + if (loadingMode !== "composition") { + lines.push(`window.__HF_SHADER_LOADING=${JSON.stringify(loadingMode)};`); + } + const script = ``; + if (/]*>/i.test(html)) + return html.replace(/]*>/i, (match) => `${match}${script}`); + if (/]*>/i.test(html)) + return html.replace(/]*>/i, (match) => `${match}${script}`); + return `${script}${html}`; +} class HyperframesPlayer extends HTMLElement { static get observedAttributes() { @@ -33,6 +141,8 @@ class HyperframesPlayer extends HTMLElement { "poster", "playback-rate", "audio-src", + SHADER_CAPTURE_SCALE_ATTR, + SHADER_LOADING_ATTR, ]; } @@ -42,6 +152,15 @@ class HyperframesPlayer extends HTMLElement { private posterEl: HTMLImageElement | null = null; private controlsApi: ReturnType | null = null; private resizeObserver: ResizeObserver; + private shaderLoaderEl: HTMLDivElement; + private shaderLoaderFillEl: HTMLDivElement; + private shaderLoaderTitleEl: HTMLSpanElement; + private shaderLoaderDetailEl: HTMLDivElement; + private shaderLoaderTransitionValueEl: HTMLSpanElement; + private shaderLoaderFrameLabelEl: HTMLSpanElement; + private shaderLoaderFrameValueEl: HTMLSpanElement; + private shaderLoaderFrameRowEl: HTMLDivElement; + private shaderLoaderHideTimeout: ReturnType | null = null; private _ready = false; private _duration = 0; @@ -141,6 +260,16 @@ class HyperframesPlayer extends HTMLElement { this.container.appendChild(this.iframe); this.shadow.appendChild(this.container); + const shaderLoader = this._createShaderLoader(); + this.shaderLoaderEl = shaderLoader.root; + this.shaderLoaderFillEl = shaderLoader.fill; + this.shaderLoaderTitleEl = shaderLoader.title; + this.shaderLoaderDetailEl = shaderLoader.detail; + this.shaderLoaderTransitionValueEl = shaderLoader.transitionValue; + this.shaderLoaderFrameLabelEl = shaderLoader.frameLabel; + this.shaderLoaderFrameValueEl = shaderLoader.frameValue; + this.shaderLoaderFrameRowEl = shaderLoader.frameRow; + this.shadow.appendChild(this.shaderLoaderEl); // Clicking the bare player surface toggles play/pause. // Ignore shadow-DOM control interactions so overlay clicks don't double-handle. @@ -167,8 +296,9 @@ class HyperframesPlayer extends HTMLElement { this._setupParentAudioFromUrl(this.getAttribute("audio-src")!); // srcdoc wins over src per HTML spec when both are set; mirror both attributes // so the browser applies the standard precedence rules. - if (this.hasAttribute("srcdoc")) this.iframe.srcdoc = this.getAttribute("srcdoc")!; - if (this.hasAttribute("src")) this.iframe.src = this.getAttribute("src")!; + if (this.hasAttribute("srcdoc")) + this.iframe.srcdoc = this._prepareSrcdoc(this.getAttribute("srcdoc")!); + if (this.hasAttribute("src")) this.iframe.src = this._prepareSrc(this.getAttribute("src")!); } disconnectedCallback() { @@ -176,6 +306,8 @@ class HyperframesPlayer extends HTMLElement { window.removeEventListener("message", this._onMessage); this.iframe.removeEventListener("load", this._onIframeLoad); if (this._probeInterval) clearInterval(this._probeInterval); + if (this.shaderLoaderHideTimeout) clearTimeout(this.shaderLoaderHideTimeout); + this.shaderLoaderHideTimeout = null; this._teardownMediaObserver(); this.controlsApi?.destroy(); for (const m of this._parentMedia) { @@ -190,7 +322,7 @@ class HyperframesPlayer extends HTMLElement { case "src": if (val) { this._ready = false; - this.iframe.src = val; + this.iframe.src = this._prepareSrc(val); } break; case "srcdoc": @@ -198,7 +330,7 @@ class HyperframesPlayer extends HTMLElement { // srcdoc and let src take over. Always reset readiness; the iframe will // load a new document either way. this._ready = false; - if (val !== null) this.iframe.srcdoc = val; + if (val !== null) this.iframe.srcdoc = this._prepareSrcdoc(val); else this.iframe.removeAttribute("srcdoc"); break; case "width": @@ -234,6 +366,10 @@ class HyperframesPlayer extends HTMLElement { case "audio-src": if (val) this._setupParentAudioFromUrl(val); break; + case SHADER_CAPTURE_SCALE_ATTR: + case SHADER_LOADING_ATTR: + this._reloadShaderOptions(); + break; } } @@ -355,6 +491,21 @@ class HyperframesPlayer extends HTMLElement { this.setAttribute("playback-rate", String(r)); } + get shaderCaptureScale() { + return Number(normalizeShaderCaptureScale(this.getAttribute(SHADER_CAPTURE_SCALE_ATTR)) ?? "1"); + } + set shaderCaptureScale(scale: number) { + this.setAttribute(SHADER_CAPTURE_SCALE_ATTR, String(scale)); + } + + get shaderLoading() { + return normalizeShaderLoadingMode(this.getAttribute(SHADER_LOADING_ATTR)); + } + set shaderLoading(mode: ShaderLoadingMode) { + if (mode === "composition") this.removeAttribute(SHADER_LOADING_ATTR); + else this.setAttribute(SHADER_LOADING_ATTR, mode); + } + get muted() { return this.hasAttribute("muted"); } @@ -384,6 +535,236 @@ class HyperframesPlayer extends HTMLElement { } } + private _shaderCaptureScaleParam(): string | null { + return normalizeShaderCaptureScale(this.getAttribute(SHADER_CAPTURE_SCALE_ATTR)); + } + + private _shaderLoadingMode(): ShaderLoadingMode { + return normalizeShaderLoadingMode(this.getAttribute(SHADER_LOADING_ATTR)); + } + + private _prepareSrc(src: string): string { + return withShaderQueryParams(src, this._shaderCaptureScaleParam(), this._shaderLoadingMode()); + } + + private _prepareSrcdoc(srcdoc: string): string { + return injectShaderOptionsIntoSrcdoc( + srcdoc, + this._shaderCaptureScaleParam(), + this._shaderLoadingMode(), + ); + } + + private _reloadShaderOptions(): void { + if (this._shaderLoadingMode() !== "player") { + this._resetShaderLoader(); + } + if (this.hasAttribute("srcdoc")) { + this.iframe.srcdoc = this._prepareSrcdoc(this.getAttribute("srcdoc") || ""); + return; + } + if (this.hasAttribute("src")) { + this.iframe.src = this._prepareSrc(this.getAttribute("src") || ""); + } + } + + private _createShaderLoader(): ShaderLoaderElements { + const root = document.createElement("div"); + root.className = "hfp-shader-loader"; + root.setAttribute("role", "status"); + root.setAttribute("aria-live", "polite"); + root.setAttribute("aria-label", "Preparing scene transitions"); + root.setAttribute("data-hyperframes-ignore", ""); + root.draggable = false; + + const blockOverlayInteraction = (event: Event) => { + event.preventDefault(); + event.stopPropagation(); + }; + for (const eventName of [ + "selectstart", + "dragstart", + "pointerdown", + "mousedown", + "click", + "dblclick", + "contextmenu", + "touchstart", + ]) { + root.addEventListener(eventName, blockOverlayInteraction, { capture: true }); + } + + const panel = document.createElement("div"); + panel.className = "hfp-shader-loader-panel"; + panel.draggable = false; + + const markFrame = document.createElement("div"); + markFrame.className = "hfp-shader-loader-mark"; + markFrame.draggable = false; + markFrame.innerHTML = [ + '", + ].join(""); + + const title = document.createElement("div"); + title.className = "hfp-shader-loader-title"; + const titleText = document.createElement("span"); + titleText.className = "hfp-shader-loader-title-text"; + titleText.textContent = SHADER_LOADING_PHRASES[0] || "Preparing scene transitions"; + title.appendChild(titleText); + + const detail = document.createElement("div"); + detail.className = "hfp-shader-loader-detail"; + detail.textContent = "Rendering animated scene samples for shader transitions."; + + const track = document.createElement("div"); + track.className = "hfp-shader-loader-track"; + track.setAttribute("aria-hidden", "true"); + const fill = document.createElement("div"); + fill.className = "hfp-shader-loader-fill"; + track.appendChild(fill); + + const progress = document.createElement("div"); + progress.className = "hfp-shader-loader-progress"; + const createProgressRow = (labelText: string) => { + const row = document.createElement("div"); + row.className = "hfp-shader-loader-row"; + const label = document.createElement("span"); + label.className = "hfp-shader-loader-label"; + label.textContent = labelText; + const value = document.createElement("span"); + value.className = "hfp-shader-loader-value"; + row.appendChild(label); + row.appendChild(value); + progress.appendChild(row); + return { row, label, value }; + }; + const transitionStatus = createProgressRow("transition"); + const frameStatus = createProgressRow("transition frame"); + + panel.appendChild(markFrame); + panel.appendChild(title); + panel.appendChild(detail); + panel.appendChild(track); + panel.appendChild(progress); + root.appendChild(panel); + + return { + root, + fill, + title: titleText, + detail, + transitionValue: transitionStatus.value, + frameLabel: frameStatus.label, + frameValue: frameStatus.value, + frameRow: frameStatus.row, + }; + } + + private _showShaderLoader(): void { + if (this.shaderLoaderHideTimeout) { + clearTimeout(this.shaderLoaderHideTimeout); + this.shaderLoaderHideTimeout = null; + } + this.shaderLoaderEl.classList.remove("hfp-hiding"); + this.shaderLoaderEl.classList.add("hfp-visible"); + } + + private _hideShaderLoader(): void { + if (this.shaderLoaderEl.classList.contains("hfp-hiding")) { + if (!this.shaderLoaderHideTimeout) this._scheduleShaderLoaderHideCleanup(); + return; + } + if (!this.shaderLoaderEl.classList.contains("hfp-visible")) return; + this.shaderLoaderEl.classList.add("hfp-hiding"); + this.shaderLoaderEl.classList.remove("hfp-visible"); + this._scheduleShaderLoaderHideCleanup(); + } + + private _scheduleShaderLoaderHideCleanup(): void { + if (this.shaderLoaderHideTimeout) clearTimeout(this.shaderLoaderHideTimeout); + this.shaderLoaderHideTimeout = setTimeout(() => { + this.shaderLoaderEl.classList.remove("hfp-hiding"); + this.shaderLoaderHideTimeout = null; + }, 420); + } + + private _resetShaderLoader(): void { + if (this.shaderLoaderHideTimeout) { + clearTimeout(this.shaderLoaderHideTimeout); + this.shaderLoaderHideTimeout = null; + } + this.shaderLoaderEl.classList.remove("hfp-visible", "hfp-hiding"); + this.shaderLoaderFillEl.style.transform = "scaleX(0)"; + this.shaderLoaderTransitionValueEl.textContent = ""; + this.shaderLoaderFrameValueEl.textContent = ""; + this.shaderLoaderFrameRowEl.style.visibility = "hidden"; + } + + private _updateShaderLoader(status: ShaderTransitionState): void { + if (this._shaderLoadingMode() !== "player") { + this._resetShaderLoader(); + return; + } + if (status.ready || !status.loading) { + this._hideShaderLoader(); + return; + } + + const progress = + typeof status.progress === "number" && Number.isFinite(status.progress) ? status.progress : 0; + const total = + typeof status.total === "number" && Number.isFinite(status.total) ? status.total : 0; + const ratio = total > 0 ? Math.min(1, Math.max(0, progress / total)) : 0; + const phraseIndex = Math.min( + SHADER_LOADING_PHRASES.length - 1, + Math.floor(ratio * SHADER_LOADING_PHRASES.length), + ); + this.shaderLoaderTitleEl.textContent = + SHADER_LOADING_PHRASES[phraseIndex] || "Preparing scene transitions"; + this.shaderLoaderDetailEl.textContent = + status.phase === "cached" + ? "Loading cached transition frames before playback." + : status.phase === "finalizing" + ? "Uploading transition textures for smooth playback." + : "Rendering animated scene samples for shader transitions."; + this.shaderLoaderFillEl.style.transform = `scaleX(${ratio})`; + + this.shaderLoaderTransitionValueEl.textContent = + status.currentTransition !== undefined && status.transitionTotal !== undefined + ? `${status.currentTransition}/${status.transitionTotal}` + : total > 0 + ? `${progress}/${total}` + : ""; + + const frameValue = + status.transitionFrame !== undefined && status.transitionFrames !== undefined + ? `${status.transitionFrame}/${status.transitionFrames}` + : ""; + this.shaderLoaderFrameLabelEl.textContent = + status.phase === "cached" + ? "cached transition frames" + : status.phase === "finalizing" + ? "finalizing transition frames" + : "rendering transition frames"; + this.shaderLoaderFrameValueEl.textContent = frameValue; + this.shaderLoaderFrameRowEl.style.visibility = frameValue ? "visible" : "hidden"; + this.shaderLoaderEl.setAttribute("aria-valuenow", String(Math.round(ratio * 100))); + this._showShaderLoader(); + } + /** * Reach into the runtime's `window.__player.seek` directly, skipping the * postMessage hop. Same-origin only — cross-origin embeds throw a @@ -426,6 +807,18 @@ class HyperframesPlayer extends HTMLElement { const data = e.data; if (!data || data.source !== "hf-preview") return; + if (data.type === "shader-transition-state") { + const state: ShaderTransitionState = + data.state && typeof data.state === "object" ? data.state : {}; + this._updateShaderLoader(state); + this.dispatchEvent( + new CustomEvent("shadertransitionstate", { + detail: { compositionId: data.compositionId, state }, + }), + ); + return; + } + if (data.type === "state") { this._currentTime = (data.frame ?? 0) / DEFAULT_FPS; const wasPlaying = !this._paused; @@ -501,6 +894,7 @@ class HyperframesPlayer extends HTMLElement { private _onIframeLoad() { let attempts = 0; this._runtimeInjected = false; + this._resetShaderLoader(); // A fresh iframe means a fresh runtime — `mediaOutputMuted` and the // autoplay-blocked latch are both reset inside it. The web component's // `_audioOwner` must reset to match, otherwise a composition switch on diff --git a/packages/player/src/styles.ts b/packages/player/src/styles.ts index 35a565b42..200269e77 100644 --- a/packages/player/src/styles.ts +++ b/packages/player/src/styles.ts @@ -31,6 +31,161 @@ export const PLAYER_STYLES = /* css */ ` pointer-events: none; } + .hfp-shader-loader { + position: absolute; + inset: 0; + z-index: 20; + display: grid; + place-items: center; + visibility: hidden; + opacity: 0; + pointer-events: none; + background: #030504; + color: #f4f7fb; + cursor: default; + user-select: none; + -webkit-user-select: none; + transition: opacity 420ms ease-out, visibility 420ms ease-out; + } + + .hfp-shader-loader.hfp-visible, + .hfp-shader-loader.hfp-hiding { + visibility: visible; + } + + .hfp-shader-loader.hfp-visible { + opacity: 1; + pointer-events: auto; + } + + .hfp-shader-loader.hfp-hiding { + opacity: 0; + pointer-events: none; + } + + .hfp-shader-loader-panel { + display: grid; + grid-template-rows: 86px 40px 26px 12px 44px; + justify-items: center; + align-items: center; + gap: 8px; + width: min(620px, 82%); + text-align: center; + font-family: Inter, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; + } + + .hfp-shader-loader-mark { + width: 86px; + height: 86px; + display: grid; + place-items: center; + overflow: visible; + } + + .hfp-shader-loader-mark svg { + display: block; + overflow: visible; + filter: drop-shadow(0 0 5px rgba(79, 219, 94, 0.16)); + pointer-events: none; + } + + .hfp-shader-loader-title { + width: 100%; + height: 40px; + overflow: hidden; + white-space: nowrap; + text-overflow: ellipsis; + font-size: 26px; + line-height: 40px; + font-weight: 700; + letter-spacing: 0; + } + + .hfp-shader-loader-title-text { + color: transparent; + background: linear-gradient( + 90deg, + rgba(244, 247, 251, 0.84) 0%, + #ffffff 42%, + #80efe4 52%, + #ffffff 62%, + rgba(244, 247, 251, 0.84) 100% + ); + background-size: 220% 100%; + -webkit-background-clip: text; + background-clip: text; + animation: hfp-shader-loader-sheen 1.9s linear infinite; + } + + .hfp-shader-loader-detail { + width: 100%; + height: 26px; + overflow: hidden; + white-space: nowrap; + text-overflow: ellipsis; + color: rgba(244, 247, 251, 0.62); + font-size: 15px; + line-height: 26px; + font-weight: 500; + } + + .hfp-shader-loader-track { + width: min(360px, 100%); + height: 8px; + overflow: hidden; + border-radius: 999px; + background: rgba(255, 255, 255, 0.1); + } + + .hfp-shader-loader-fill { + width: 100%; + height: 100%; + border-radius: inherit; + background: linear-gradient(90deg, #06e3fa, #4fdb5e); + transform: scaleX(0); + transform-origin: left center; + transition: transform 160ms ease; + } + + .hfp-shader-loader-progress { + width: min(420px, 100%); + height: 44px; + display: grid; + grid-template-rows: repeat(2, 22px); + color: rgba(244, 247, 251, 0.48); + font: 600 13px/22px "IBM Plex Mono", "SF Mono", "Fira Code", "Courier New", monospace; + font-variant-numeric: tabular-nums; + } + + .hfp-shader-loader-row { + display: grid; + grid-template-columns: minmax(0, 1fr) 74px; + align-items: center; + column-gap: 20px; + width: 100%; + white-space: nowrap; + } + + .hfp-shader-loader-label { + min-width: 0; + overflow: hidden; + text-align: left; + text-overflow: ellipsis; + } + + .hfp-shader-loader-value { + text-align: right; + } + + @keyframes hfp-shader-loader-sheen { + from { + background-position: 140% 0; + } + to { + background-position: -140% 0; + } + } + /* ── Theming via CSS custom properties ── * * Override from outside the shadow DOM: diff --git a/packages/shader-transitions/README.md b/packages/shader-transitions/README.md index 050ca63f0..d42ddfc11 100644 --- a/packages/shader-transitions/README.md +++ b/packages/shader-transitions/README.md @@ -30,7 +30,7 @@ const tl = init({ }); ``` -The `init()` function captures each scene to a WebGL texture at transition time, crossfades between them using the selected shader, and returns a GSAP timeline. If WebGL is unavailable, it falls back to hard cuts. +The `init()` function pre-captures animated scene samples for every transition, composites cached samples with the selected shader during playback, and returns a GSAP timeline. Scene animations keep advancing through shader transitions without running DOM captures in the playback loop. If WebGL is unavailable, it falls back to normal timeline playback without shader compositing. ### With an existing timeline @@ -74,14 +74,19 @@ init({ ### `init(config): GsapTimeline` -| Option | Type | Required | Description | -| --------------- | -------------------- | -------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `bgColor` | `string` | yes | Fallback background color (hex) for scene capture. Use the composition's body/canvas background — individual scenes set their own `background-color` via CSS. | -| `accentColor` | `string` | no | Accent color (hex) for shader glow effects | -| `scenes` | `string[]` | yes | Element IDs of each scene, in order | -| `transitions` | `TransitionConfig[]` | yes | Transition definitions (see below) | -| `timeline` | `GsapTimeline` | no | Existing timeline to attach transitions to | -| `compositionId` | `string` | no | Override the `data-composition-id` for timeline registration | +| Option | Type | Required | Description | +| ------------------- | -------------------- | -------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `bgColor` | `string` | yes | Fallback background color (hex) for scene capture. Use the composition's body/canvas background — individual scenes set their own `background-color` via CSS. | +| `accentColor` | `string` | no | Accent color (hex) for shader glow effects | +| `scenes` | `string[]` | yes | Element IDs of each scene, in order | +| `transitions` | `TransitionConfig[]` | yes | Transition definitions (see below) | +| `timeline` | `GsapTimeline` | no | Existing timeline to attach transitions to | +| `compositionId` | `string` | no | Override the `data-composition-id` for timeline registration | +| `previewCaptureFps` | `number` | no | Browser preview pre-capture samples per transition second. Defaults to `30`; rendering uses deterministic per-frame compositing instead. | + +Browser preview capture scale and transition-prep loading UI ownership are controlled by `` (`shader-capture-scale`, `shader-loading`) instead of composition code. Direct non-player previews keep the built-in full-fidelity loading fallback. + +Browser previews store captured transition snapshots in IndexedDB using a key derived from composition ID, scene DOM/style signatures, transition timing, capture FPS, scale, and dimensions. On refresh, matching snapshots are reloaded into WebGL textures instead of being captured again. Runtime scene or stylesheet edits mark only adjacent transition caches dirty; recapture is deferred until playback so editing stays responsive. ### `TransitionConfig` diff --git a/packages/shader-transitions/src/capture.ts b/packages/shader-transitions/src/capture.ts index fc3a26874..5b983d16b 100644 --- a/packages/shader-transitions/src/capture.ts +++ b/packages/shader-transitions/src/capture.ts @@ -2,6 +2,22 @@ import html2canvas from "html2canvas"; import { DEFAULT_WIDTH, DEFAULT_HEIGHT } from "./webgl.js"; let patched = false; +const VOID_ELEMENT_TAGS = new Set([ + "AREA", + "BASE", + "BR", + "COL", + "EMBED", + "HR", + "IMG", + "INPUT", + "LINK", + "META", + "PARAM", + "SOURCE", + "TRACK", + "WBR", +]); function patchCreatePattern(): void { if (patched) return; @@ -27,94 +43,113 @@ export function initCapture(): void { patchCreatePattern(); } +export interface CaptureSceneOptions { + forceVisible?: boolean; + preferBrowserPaint?: boolean; + scale?: number; +} + +function forceSceneVisibleInClone(source: HTMLElement, cloneDoc: Document): void { + if (!source.id) return; + const clone = cloneDoc.getElementById(source.id); + if (!(clone instanceof HTMLElement)) return; + + clone.style.opacity = "1"; + clone.style.visibility = "visible"; + clone.querySelectorAll("[data-start]").forEach((el) => { + el.style.visibility = "visible"; + }); +} + +function stabilizeTransformedBoxShadows(root: HTMLElement): void { + const view = root.ownerDocument.defaultView; + if (!view) return; + + [root, ...Array.from(root.querySelectorAll("*"))].forEach((el) => { + if (VOID_ELEMENT_TAGS.has(el.tagName)) return; + const styles = view.getComputedStyle(el); + if (styles.boxShadow === "none" || styles.transform === "none") return; + + const shadow = root.ownerDocument.createElement("div"); + shadow.setAttribute("data-hyper-shader-shadow-shim", ""); + shadow.style.cssText = [ + "position:absolute", + "inset:0", + "border-radius:inherit", + `box-shadow:${styles.boxShadow}`, + "background:transparent", + "pointer-events:none", + "z-index:0", + ].join(";"); + + if (styles.position === "static") { + el.style.position = "relative"; + } + el.style.boxShadow = "none"; + el.insertBefore(shadow, el.firstChild); + }); +} + export function captureScene( sceneEl: HTMLElement, bgColor: string, width: number = DEFAULT_WIDTH, height: number = DEFAULT_HEIGHT, + options: CaptureSceneOptions = {}, ): Promise { - return html2canvas(sceneEl, { - width, - height, - scale: 1, - backgroundColor: bgColor, - logging: false, - // Safari applies stricter canvas-taint rules than Chrome. SVG data URLs - // with elements (e.g. feTurbulence grain backgrounds), certain - // cross-origin images, and mask/clip-path url() refs can taint the - // output canvas on WebKit. Without these flags, html2canvas throws - // `SecurityError: The operation is insecure` during its own read-back - // path and every shader transition falls through to the catch handler - // — observed in Safari + Claude Design's cross-origin iframe sandbox. - // - // useCORS: send CORS headers on image fetches so cross-origin images - // with proper `Access-Control-Allow-Origin` don't taint the - // canvas in the first place. Strict improvement. - // allowTaint: let html2canvas complete and return a canvas even when it - // becomes tainted (instead of throwing). Important caveat: - // a tainted canvas CANNOT be uploaded to WebGL via - // `gl.texImage2D` — WebGL spec requires SecurityError on - // non-origin-clean sources, with no opt-out. So this flag - // only moves the failure point from html2canvas to the - // texImage2D call in webgl.ts. In both cases `hyper-shader.ts` - // catches the rejected promise and runs the CSS crossfade - // fallback. Net effect: the end-user UX is the same (smooth - // CSS fade in either case), but we get a cleaner, more - // predictable error site and the flag is defensively - // correct for the non-taint branches where it genuinely - // helps (e.g., `crossOrigin="anonymous"` image fetches - // that already had CORS headers). - useCORS: true, - allowTaint: true, - ignoreElements: (el: Element) => el.tagName === "CANVAS" || el.hasAttribute("data-no-capture"), - }); -} - -/** - * Capture the incoming scene with .scene-content hidden (background + decoratives only). - * Shows the scene behind the outgoing scene via z-index, waits 2 rAFs for font rendering, - * captures, then restores. - * - * IMPORTANT: We force `visibility: visible` during capture because the HyperFrames runtime's - * time-based visibility gate (in `packages/core/src/runtime/init.ts`) sets `style.visibility - * = "hidden"` on every `[data-start]` element that's outside its current playback window — - * every frame. When a shader transition fires *before* the incoming scene's `data-start` - * boundary (the recommended "transition.time = boundary - duration/2" centered placement), - * the runtime has `visibility: hidden` on the incoming scene. Without the visibility override - * here, `html2canvas` captures the element as blank → shader transitions from the real - * outgoing scene to a blank incoming texture → users see content fade/morph into the - * background color mid-transition (a visible "blink"). Forcing `visibility: visible` only - * for the duration of the capture fixes this without affecting what the user sees during - * normal playback. - */ -export function captureIncomingScene( - toScene: HTMLElement, - bgColor: string, - width: number = DEFAULT_WIDTH, - height: number = DEFAULT_HEIGHT, -): Promise { - return new Promise((resolve, reject) => { - const origZ = toScene.style.zIndex; - const origOpacity = toScene.style.opacity; - const origVisibility = toScene.style.visibility; - toScene.style.zIndex = "-1"; - toScene.style.opacity = "1"; - toScene.style.visibility = "visible"; - - const contentEl = toScene.querySelector(".scene-content"); - if (contentEl) contentEl.style.visibility = "hidden"; - - const restore = () => { - if (contentEl) contentEl.style.visibility = ""; - toScene.style.visibility = origVisibility; - toScene.style.opacity = origOpacity; - toScene.style.zIndex = origZ; - }; - - requestAnimationFrame(() => { - requestAnimationFrame(() => { - captureScene(toScene, bgColor, width, height).then(resolve, reject).finally(restore); - }); + const captureWithRenderer = (foreignObjectRendering: boolean): Promise => { + return html2canvas(sceneEl, { + width, + height, + scale: options.scale ?? 1, + backgroundColor: bgColor, + logging: false, + foreignObjectRendering, + // Safari applies stricter canvas-taint rules than Chrome. SVG data URLs + // with elements (e.g. feTurbulence grain backgrounds), certain + // cross-origin images, and mask/clip-path url() refs can taint the + // output canvas on WebKit. Without these flags, html2canvas throws + // `SecurityError: The operation is insecure` during its own read-back + // path and every shader transition falls through to the catch handler + // — observed in Safari + Claude Design's cross-origin iframe sandbox. + // + // useCORS: send CORS headers on image fetches so cross-origin images + // with proper `Access-Control-Allow-Origin` don't taint the + // canvas in the first place. Strict improvement. + // allowTaint: let html2canvas complete and return a canvas even when it + // becomes tainted (instead of throwing). Important caveat: + // a tainted canvas CANNOT be uploaded to WebGL via + // `gl.texImage2D` — WebGL spec requires SecurityError on + // non-origin-clean sources, with no opt-out. So this flag + // only moves the failure point from html2canvas to the + // texImage2D call in webgl.ts. The caller catches the + // rejected promise and keeps the DOM fallback visible. Net + // effect: the end-user UX avoids blank frames either way, + // but we get a cleaner, more predictable error site and the + // flag is defensively correct for the non-taint branches + // where it genuinely helps (e.g., + // `crossOrigin="anonymous"` image fetches that already had + // CORS headers). + useCORS: true, + allowTaint: true, + onclone: (cloneDoc) => { + if (!sceneEl.id) return; + const clone = cloneDoc.getElementById(sceneEl.id); + if (clone instanceof HTMLElement) { + stabilizeTransformedBoxShadows(clone); + } + if (options.forceVisible) { + forceSceneVisibleInClone(sceneEl, cloneDoc); + } + }, + ignoreElements: (el: Element) => + el.tagName === "CANVAS" || el.hasAttribute("data-no-capture"), }); - }); + }; + + if (options.preferBrowserPaint === true) { + return captureWithRenderer(true).catch(() => captureWithRenderer(false)); + } + + return captureWithRenderer(false); } diff --git a/packages/shader-transitions/src/hyper-shader.ts b/packages/shader-transitions/src/hyper-shader.ts index 64685b083..8bb3171d8 100644 --- a/packages/shader-transitions/src/hyper-shader.ts +++ b/packages/shader-transitions/src/hyper-shader.ts @@ -2,15 +2,16 @@ import { createContext, setupQuad, createProgram, + createProgramWithVertex, createTexture, - uploadTexture, + uploadTextureSource, renderShader, DEFAULT_WIDTH, DEFAULT_HEIGHT, type AccentColors, } from "./webgl.js"; import { getFragSource, type ShaderName } from "./shaders/registry.js"; -import { initCapture, captureScene, captureIncomingScene } from "./capture.js"; +import { initCapture, captureScene } from "./capture.js"; declare const gsap: { timeline: (opts: Record) => GsapTimeline; @@ -24,9 +25,13 @@ declare const gsap: { interface GsapTimeline { paused: () => boolean; - play: () => GsapTimeline; - pause: () => GsapTimeline; - time: () => number; + play: (from?: number, suppressEvents?: boolean) => GsapTimeline; + pause: (atTime?: number, suppressEvents?: boolean) => GsapTimeline; + time: { + (): number; + (value: number, suppressEvents?: boolean): GsapTimeline; + }; + seek?: (position: number | string, suppressEvents?: boolean) => GsapTimeline; call: (fn: () => void, args: null, position: number) => GsapTimeline; to: ( target: Record, @@ -58,14 +63,84 @@ export interface HyperShaderConfig { transitions: TransitionConfig[]; timeline?: GsapTimeline; compositionId?: string; + previewCaptureFps?: number; } interface TransState { active: boolean; prog: WebGLProgram | null; + progress: number; + transitionIndex: number; +} + +interface CachedTransitionFrame { + sampleIndex: number; + fromBlob: Blob | null; + toBlob: Blob | null; + fromTex: WebGLTexture | null; + toTex: WebGLTexture | null; +} + +interface TexturedTransitionFrame extends CachedTransitionFrame { + fromTex: WebGLTexture; + toTex: WebGLTexture; +} + +interface CachedTransitionFrameBlend { + a: TexturedTransitionFrame; + b: TexturedTransitionFrame; + mix: number; +} + +interface CachedTransition { + index: number; + time: number; + duration: number; fromId: string; toId: string; + prog: WebGLProgram; + frames: CachedTransitionFrame[]; + cacheKey: string; + dirty: boolean; + ready: boolean; + fallback: boolean; + persisted: boolean; + textureReady: boolean; + texturePromise: Promise | null; + textureGeneration: number; + textureAccess: number; + lastError?: string; +} + +interface SnapshotLoadingOverlay { + show: () => void; + update: (status: SnapshotLoadingStatus) => void; + hide: () => void; +} + +interface SnapshotLoadingStatus { progress: number; + total: number; + currentTransition?: number; + transitionTotal?: number; + transitionFrame?: number; + transitionFrames?: number; + phase?: "cached" | "capturing" | "finalizing"; +} + +interface SnapshotCacheEntry { + key: string; + blob: Blob; + width: number; + height: number; + updatedAt: number; +} + +interface SceneStyleState { + scene: HTMLElement | null; + opacity: string; + visibility: string; + pointerEvents: string; } // Defaults for transition duration/ease. Used by every fallback site in this @@ -75,6 +150,23 @@ interface TransState { // producer reads to plan compositing. const DEFAULT_DURATION = 0.7; const DEFAULT_EASE = "power2.inOut"; +const NO_FLIP_VERT_SRC = + "attribute vec2 a_pos; varying vec2 v_uv; void main(){" + + "v_uv=a_pos*0.5+0.5; gl_Position=vec4(a_pos,0,1);}"; +const SNAPSHOT_LOADING_PHRASES = [ + "Preparing scene transitions", + "Sampling outgoing scene motion", + "Sampling incoming scene motion", + "Caching transition frames", + "Finalizing transition preview", +]; +const SNAPSHOT_CACHE_DB = "hyper-shader-preview-cache"; +const SNAPSHOT_CACHE_STORE = "frames"; +const SNAPSHOT_CACHE_VERSION = 1; +const SNAPSHOT_CACHE_SCHEMA = "v1"; +const MAX_TEXTURED_TRANSITIONS = 2; +const TEXTURE_PRELOAD_LOOKAHEAD_SECONDS = 1.25; +const MAX_SNAPSHOT_CACHE_ENTRIES = 1200; function parseHex(hex: string): [number, number, number] { const h = hex.replace("#", ""); @@ -95,6 +187,603 @@ function deriveAccentColors(hex: string): AccentColors { }; } +function clampNumber(value: number, min: number, max: number): number { + return Math.min(max, Math.max(min, value)); +} + +function resolvePositiveNumber(value: number | undefined, fallback: number): number { + if (value === undefined) return fallback; + return Number.isFinite(value) && value > 0 ? value : fallback; +} + +const PLAYER_CAPTURE_SCALE_PARAM = "__hf_shader_capture_scale"; +const PLAYER_LOADING_PARAM = "__hf_shader_loading"; + +function readPlayerOption(globalName: string, queryName: string): string | null { + const globalValue = (window as unknown as Record)[globalName]; + if (typeof globalValue === "string") return globalValue; + if (typeof globalValue === "number" && Number.isFinite(globalValue)) return String(globalValue); + try { + return new URLSearchParams(window.location.search).get(queryName); + } catch { + return null; + } +} + +function resolvePlayerCaptureScale(): number { + const raw = readPlayerOption("__HF_SHADER_CAPTURE_SCALE", PLAYER_CAPTURE_SCALE_PARAM); + const parsed = raw === null ? NaN : Number(raw); + return clampNumber(Number.isFinite(parsed) && parsed > 0 ? parsed : 1, 0.25, 1); +} + +function resolvePlayerLoadingMode(): "internal" | "player" | "none" { + const raw = readPlayerOption("__HF_SHADER_LOADING", PLAYER_LOADING_PARAM)?.trim().toLowerCase(); + if (raw === "player" || raw === "true" || raw === "1") return "player"; + if (raw === "none" || raw === "false" || raw === "0" || raw === "off") return "none"; + return "internal"; +} + +function stableHash(input: string): string { + let hash = 2166136261; + for (let i = 0; i < input.length; i += 1) { + hash ^= input.charCodeAt(i); + hash = Math.imul(hash, 16777619); + } + return (hash >>> 0).toString(16).padStart(8, "0"); +} + +function getDocumentStyleSignature(doc: Document): string { + const styleText = Array.from(doc.querySelectorAll("style")) + .map((style) => style.textContent || "") + .join("\n"); + const linkedStyles = Array.from(doc.querySelectorAll('link[rel~="stylesheet"]')) + .map((link) => `${link.href}:${link.getAttribute("integrity") || ""}`) + .join("\n"); + return stableHash(`${styleText}\n${linkedStyles}`); +} + +function getDocumentScriptSignature(doc: Document): string { + const projectSignature = Array.from( + doc.querySelectorAll('meta[name="hyperframes-project-signature"]'), + ) + .map((meta) => meta.getAttribute("content") || "") + .join("\n"); + const scriptText = Array.from(doc.querySelectorAll("script")) + .map((script) => { + const attrs = [ + script.type, + script.src, + script.getAttribute("integrity") || "", + script.getAttribute("crossorigin") || "", + script.getAttribute("data-hyperframes-runtime") || "", + ].join(":"); + return `${attrs}\n${script.src ? "" : script.textContent || ""}`; + }) + .join("\n"); + return stableHash(`${projectSignature}\n${scriptText}`); +} + +function getSceneSignature(sceneId: string): string { + const scene = document.getElementById(sceneId); + if (!scene) return "missing"; + return stableHash( + `${getDocumentStyleSignature(document)}\n${getDocumentScriptSignature(document)}\n${getSceneSignatureHtml(scene)}`, + ); +} + +function removeStyleProperties(el: HTMLElement, properties: string[]): void { + for (const property of properties) { + el.style.removeProperty(property); + } + if (el.getAttribute("style")?.trim() === "") { + el.removeAttribute("style"); + } +} + +function getSceneSignatureHtml(scene: HTMLElement): string { + const clone = scene.cloneNode(true) as HTMLElement; + + // HyperShader and the core runtime mutate these inline styles during seek and + // playback. Cache identity should track authored content, not the last preview + // playhead state. + removeStyleProperties(clone, ["opacity", "visibility", "pointer-events"]); + clone.querySelectorAll("[data-start]").forEach((el) => { + removeStyleProperties(el, ["visibility"]); + }); + + return clone.outerHTML; +} + +function makeSnapshotKey(cacheKey: string, sampleIndex: number, side: "from" | "to"): string { + return `${cacheKey}:sample:${sampleIndex}:${side}`; +} + +function openSnapshotDb(): Promise { + if (typeof indexedDB === "undefined") return Promise.resolve(null); + return new Promise((resolve) => { + const request = indexedDB.open(SNAPSHOT_CACHE_DB, SNAPSHOT_CACHE_VERSION); + request.onupgradeneeded = () => { + const db = request.result; + if (!db.objectStoreNames.contains(SNAPSHOT_CACHE_STORE)) { + db.createObjectStore(SNAPSHOT_CACHE_STORE, { keyPath: "key" }); + } + }; + request.onerror = () => resolve(null); + request.onblocked = () => resolve(null); + request.onsuccess = () => { + const db = request.result; + db.onversionchange = () => { + db.close(); + snapshotDbPromise = null; + }; + resolve(db); + }; + }); +} + +let snapshotDbPromise: Promise | null = null; + +function resetSnapshotDb(db: IDBDatabase | null): void { + try { + db?.close(); + } catch { + // Ignore close failures; the next cache operation will reopen the DB. + } + snapshotDbPromise = null; +} + +function getSnapshotDb(): Promise { + snapshotDbPromise = snapshotDbPromise || openSnapshotDb(); + return snapshotDbPromise; +} + +async function getSnapshotEntry(key: string): Promise { + const db = await getSnapshotDb(); + if (!db) return null; + return new Promise((resolve) => { + try { + const tx = db.transaction(SNAPSHOT_CACHE_STORE, "readonly"); + const request = tx.objectStore(SNAPSHOT_CACHE_STORE).get(key); + request.onerror = () => resolve(null); + request.onsuccess = () => { + const result = request.result; + if ( + result && + typeof result === "object" && + "blob" in result && + result.blob instanceof Blob + ) { + resolve(result as SnapshotCacheEntry); + } else { + resolve(null); + } + }; + } catch { + resetSnapshotDb(db); + resolve(null); + } + }); +} + +async function putSnapshotEntry(entry: SnapshotCacheEntry): Promise { + const db = await getSnapshotDb(); + if (!db) return false; + return new Promise((resolve) => { + try { + const tx = db.transaction(SNAPSHOT_CACHE_STORE, "readwrite"); + tx.oncomplete = () => resolve(true); + tx.onerror = () => resolve(false); + tx.objectStore(SNAPSHOT_CACHE_STORE).put(entry); + } catch { + resetSnapshotDb(db); + resolve(false); + } + }); +} + +function canvasToPngBlob(canvas: HTMLCanvasElement): Promise { + return new Promise((resolve) => { + try { + canvas.toBlob((blob) => resolve(blob), "image/png"); + } catch { + resolve(null); + } + }); +} + +async function pruneSnapshotCache( + compId: string, + activeCacheKeys: Set, + maxEntries: number = MAX_SNAPSHOT_CACHE_ENTRIES, +): Promise { + const db = await getSnapshotDb(); + if (!db) return; + const entries = await new Promise>((resolve) => { + try { + const tx = db.transaction(SNAPSHOT_CACHE_STORE, "readonly"); + const request = tx.objectStore(SNAPSHOT_CACHE_STORE).getAll(); + request.onerror = () => resolve([]); + request.onsuccess = () => { + const rows: Array<{ key: string; updatedAt: number }> = []; + for (const result of request.result) { + if ( + result && + typeof result === "object" && + "key" in result && + typeof result.key === "string" + ) { + const updatedAt = + "updatedAt" in result && typeof result.updatedAt === "number" ? result.updatedAt : 0; + rows.push({ key: result.key, updatedAt }); + } + } + resolve(rows); + }; + } catch { + resetSnapshotDb(db); + resolve([]); + } + }); + const projectPrefix = `${compId}:`; + const isActiveSnapshot = (key: string): boolean => { + for (const cacheKey of activeCacheKeys) { + if (key.startsWith(`${cacheKey}:sample:`)) return true; + } + return false; + }; + const staleProjectKeys = entries + .filter((entry) => entry.key.startsWith(projectPrefix) && !isActiveSnapshot(entry.key)) + .map((entry) => entry.key); + const staleProjectKeySet = new Set(staleProjectKeys); + const remaining = entries.filter((entry) => !staleProjectKeySet.has(entry.key)); + const activeCount = remaining.filter((entry) => isActiveSnapshot(entry.key)).length; + const removable = remaining + .filter((entry) => !isActiveSnapshot(entry.key)) + .sort((a, b) => b.updatedAt - a.updatedAt); + const removableBudget = Math.max(0, maxEntries - activeCount); + const overflowKeys = + removable.length > removableBudget + ? removable.slice(removableBudget).map((entry) => entry.key) + : []; + const keysToDelete = Array.from(new Set([...staleProjectKeys, ...overflowKeys])); + if (keysToDelete.length === 0) return; + await new Promise((resolve) => { + try { + const tx = db.transaction(SNAPSHOT_CACHE_STORE, "readwrite"); + tx.oncomplete = () => resolve(); + tx.onerror = () => resolve(); + const store = tx.objectStore(SNAPSHOT_CACHE_STORE); + for (const key of keysToDelete) { + store.delete(key); + } + } catch { + resetSnapshotDb(db); + resolve(); + } + }); +} + +function blobToTextureSource(blob: Blob): Promise { + if (typeof createImageBitmap === "function") { + return createImageBitmap(blob); + } + + return new Promise((resolve, reject) => { + const url = URL.createObjectURL(blob); + const img = new Image(); + img.onload = () => { + URL.revokeObjectURL(url); + resolve(img); + }; + img.onerror = () => { + URL.revokeObjectURL(url); + reject(new Error("[HyperShader] Failed to decode cached snapshot")); + }; + img.src = url; + }); +} + +function closeTextureSource(source: TexImageSource): void { + if (typeof ImageBitmap !== "undefined" && source instanceof ImageBitmap) { + source.close(); + } +} + +function createRenderTexture( + gl: WebGLRenderingContext, + width: number, + height: number, +): WebGLTexture { + const tex = createTexture(gl); + gl.bindTexture(gl.TEXTURE_2D, tex); + gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, width, height, 0, gl.RGBA, gl.UNSIGNED_BYTE, null); + return tex; +} + +function createFramebuffer(gl: WebGLRenderingContext, tex: WebGLTexture): WebGLFramebuffer { + const fbo = gl.createFramebuffer(); + if (!fbo) throw new Error("[HyperShader] Failed to create framebuffer"); + gl.bindFramebuffer(gl.FRAMEBUFFER, fbo); + gl.framebufferTexture2D(gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.TEXTURE_2D, tex, 0); + gl.bindFramebuffer(gl.FRAMEBUFFER, null); + return fbo; +} + +function createSnapshotLoadingOverlay( + root: HTMLElement | null, + width: number, + height: number, +): SnapshotLoadingOverlay | null { + const doc = root?.ownerDocument || document; + const host = root || doc.body; + if (!host) return null; + + host.querySelector("[data-hyper-shader-loading]")?.remove(); + + const overlay = doc.createElement("div"); + overlay.setAttribute("data-hyper-shader-loading", ""); + overlay.setAttribute("data-hyperframes-ignore", ""); + overlay.setAttribute("data-hyperframes-picker-block", ""); + overlay.setAttribute("data-hf-ignore", ""); + overlay.setAttribute("data-no-capture", ""); + overlay.setAttribute("data-no-inspect", ""); + overlay.setAttribute("data-no-pick", ""); + overlay.setAttribute("draggable", "false"); + overlay.setAttribute("role", "status"); + overlay.setAttribute("aria-label", "Preparing scene transitions"); + overlay.style.cssText = [ + "position:absolute", + "inset:0", + `width:${width}px`, + `height:${height}px`, + "z-index:2147483647", + "display:none", + "place-items:center", + "opacity:1", + "transition:opacity 240ms ease-out", + "background:#030504", + "pointer-events:auto", + "color:#f4f7fb", + "cursor:default", + "touch-action:none", + "user-select:none", + "-webkit-user-select:none", + "-webkit-user-drag:none", + ].join(";"); + const blockOverlayInteraction = (event: Event): void => { + event.preventDefault(); + event.stopPropagation(); + }; + for (const eventName of ["selectstart", "dragstart", "pointerdown", "mousedown", "touchstart"]) { + overlay.addEventListener(eventName, blockOverlayInteraction, { capture: true }); + } + + const panel = doc.createElement("div"); + panel.setAttribute("draggable", "false"); + panel.style.cssText = [ + "display:grid", + "grid-template-rows:172px 72px 44px 44px 54px", + "justify-items:center", + "align-items:center", + "gap:0", + "width:min(1040px,82%)", + "padding:42px", + "box-sizing:border-box", + "text-align:center", + "font-family:Inter,-apple-system,BlinkMacSystemFont,'Segoe UI',sans-serif", + "user-select:none", + "-webkit-user-select:none", + "-webkit-user-drag:none", + ].join(";"); + + const markFrame = doc.createElement("div"); + markFrame.setAttribute("data-hf-loader-mark-frame", ""); + markFrame.style.cssText = [ + "width:172px", + "height:172px", + "display:grid", + "place-items:center", + "overflow:visible", + "transform-origin:50% 50%", + "will-change:transform,opacity", + "user-select:none", + "-webkit-user-select:none", + "-webkit-user-drag:none", + ].join(";"); + markFrame.innerHTML = [ + '", + ].join(""); + const mark = markFrame.querySelector("svg"); + if (!mark) return null; + mark.setAttribute("draggable", "false"); + + const phrase = doc.createElement("div"); + phrase.style.cssText = [ + "width:100%", + "height:72px", + "display:flex", + "align-items:center", + "justify-content:center", + "overflow:hidden", + "white-space:nowrap", + "text-overflow:ellipsis", + "font:600 44px/1.15 Inter,-apple-system,BlinkMacSystemFont,'Segoe UI',sans-serif", + "letter-spacing:0", + "text-align:center", + "color:#f4f7fb", + ].join(";"); + + const phraseText = doc.createElement("span"); + phraseText.textContent = SNAPSHOT_LOADING_PHRASES[0] ?? "Preparing scene transitions"; + + const detail = doc.createElement("div"); + detail.textContent = "Sampling animated scene frames so shader transitions stay in motion."; + detail.style.cssText = [ + "width:min(760px,100%)", + "height:44px", + "overflow:hidden", + "white-space:nowrap", + "text-overflow:ellipsis", + "color:rgba(244,247,251,0.64)", + "font:400 24px/1.5 Inter,-apple-system,BlinkMacSystemFont,'Segoe UI',sans-serif", + ].join(";"); + + phrase.appendChild(phraseText); + + const track = doc.createElement("div"); + track.setAttribute("aria-hidden", "true"); + track.style.cssText = [ + "width:min(520px,100%)", + "height:10px", + "overflow:hidden", + "border-radius:999px", + "background:rgba(255,255,255,0.1)", + ].join(";"); + + const fill = doc.createElement("div"); + fill.style.cssText = [ + "width:100%", + "height:100%", + "transform:scaleX(0)", + "transform-origin:left center", + "border-radius:inherit", + "background:linear-gradient(90deg,#06e3fa,#4fdb5e)", + "transition:transform 160ms ease", + ].join(";"); + + const progressText = doc.createElement("div"); + progressText.style.cssText = [ + "width:min(560px,100%)", + "height:54px", + "overflow:hidden", + "display:grid", + "grid-template-rows:repeat(2,27px)", + "font:500 18px/27px 'IBM Plex Mono','SF Mono','Fira Code',monospace", + "font-variant-numeric:tabular-nums", + "color:rgba(244,247,251,0.48)", + ].join(";"); + const createProgressRow = (labelText: string) => { + const row = doc.createElement("div"); + row.style.cssText = [ + "display:grid", + "grid-template-columns:minmax(0,1fr) auto", + "align-items:center", + "column-gap:28px", + "width:100%", + "height:27px", + "white-space:nowrap", + ].join(";"); + + const label = doc.createElement("span"); + label.textContent = labelText; + label.style.cssText = [ + "overflow:hidden", + "text-overflow:ellipsis", + "text-align:left", + "min-width:0", + ].join(";"); + + const value = doc.createElement("span"); + value.style.cssText = ["min-width:76px", "text-align:right"].join(";"); + + row.appendChild(label); + row.appendChild(value); + progressText.appendChild(row); + return { row, label, value }; + }; + const transitionStatus = createProgressRow("transition"); + const frameStatus = createProgressRow("transition frame"); + + track.appendChild(fill); + panel.appendChild(markFrame); + panel.appendChild(phrase); + panel.appendChild(detail); + panel.appendChild(track); + panel.appendChild(progressText); + overlay.appendChild(panel); + host.appendChild(overlay); + + let hideTimeout: ReturnType | null = null; + + return { + show: () => { + if (hideTimeout) { + clearTimeout(hideTimeout); + hideTimeout = null; + } + overlay.style.display = "grid"; + overlay.style.opacity = "1"; + overlay.style.pointerEvents = "auto"; + }, + update: (status: SnapshotLoadingStatus) => { + const { progress, total } = status; + const ratio = total > 0 ? clampNumber(progress / total, 0, 1) : 0; + const phraseIndex = Math.min( + SNAPSHOT_LOADING_PHRASES.length - 1, + Math.floor(ratio * SNAPSHOT_LOADING_PHRASES.length), + ); + const nextPhrase = SNAPSHOT_LOADING_PHRASES[phraseIndex] ?? "Preparing scene transitions"; + phraseText.textContent = nextPhrase; + fill.style.transform = `scaleX(${ratio})`; + const transitionValue = + status.currentTransition !== undefined && status.transitionTotal !== undefined + ? `${status.currentTransition}/${status.transitionTotal}` + : total > 0 + ? `${progress}/${total}` + : ""; + const frameValue = + status.transitionFrame !== undefined && status.transitionFrames !== undefined + ? `${status.transitionFrame}/${status.transitionFrames}` + : ""; + const phaseLabel = + status.phase === "cached" + ? "loading cached transition frames" + : status.phase === "finalizing" + ? "finalizing" + : "rendering transition frames"; + + transitionStatus.label.textContent = + status.currentTransition !== undefined ? "transition" : "transition frames"; + transitionStatus.value.textContent = transitionValue; + frameStatus.label.textContent = phaseLabel; + frameStatus.value.textContent = frameValue; + frameStatus.row.style.visibility = frameValue ? "visible" : "hidden"; + overlay.setAttribute("aria-valuenow", String(Math.round(ratio * 100))); + }, + hide: () => { + if (hideTimeout) clearTimeout(hideTimeout); + if (overlay.style.display === "none") return; + overlay.style.opacity = "0"; + overlay.style.pointerEvents = "none"; + hideTimeout = setTimeout(() => { + overlay.style.display = "none"; + overlay.style.opacity = "1"; + overlay.style.pointerEvents = "auto"; + hideTimeout = null; + }, 240); + }, + }; +} + export function init(config: HyperShaderConfig): GsapTimeline { const { bgColor, scenes, transitions } = config; @@ -183,9 +872,8 @@ export function init(config: HyperShaderConfig): GsapTimeline { const state: TransState = { active: false, prog: null, - fromId: "", - toId: "", progress: 0, + transitionIndex: -1, }; let glCanvas = document.getElementById("gl-canvas") as HTMLCanvasElement | null; @@ -221,29 +909,267 @@ export function init(config: HyperShaderConfig): GsapTimeline { } } - const textures = new Map(); - for (const id of scenes) { - textures.set(id, createTexture(gl)); - } + const canvasEl = glCanvas; + const previewCaptureFps = clampNumber(resolvePositiveNumber(config.previewCaptureFps, 30), 1, 60); + const previewCaptureScale = resolvePlayerCaptureScale(); + const loadingMode = resolvePlayerLoadingMode(); + const previewTextureWidth = Math.max(1, Math.round(compWidth * previewCaptureScale)); + const previewTextureHeight = Math.max(1, Math.round(compHeight * previewCaptureScale)); + const cachedTransitions: CachedTransition[] = []; + const blendProg = createProgramWithVertex( + gl, + NO_FLIP_VERT_SRC, + [ + "precision mediump float;", + "varying vec2 v_uv;", + "uniform sampler2D u_a;", + "uniform sampler2D u_b;", + "uniform float u_mix;", + "void main(){", + "gl_FragColor=mix(texture2D(u_a,v_uv),texture2D(u_b,v_uv),u_mix);", + "}", + ].join(""), + ); + const blendLoc = { + a: gl.getUniformLocation(blendProg, "u_a"), + b: gl.getUniformLocation(blendProg, "u_b"), + mix: gl.getUniformLocation(blendProg, "u_mix"), + pos: gl.getAttribLocation(blendProg, "a_pos"), + }; + const interpolatedFromTex = createRenderTexture(gl, previewTextureWidth, previewTextureHeight); + const interpolatedToTex = createRenderTexture(gl, previewTextureWidth, previewTextureHeight); + const interpolatedFromFbo = createFramebuffer(gl, interpolatedFromTex); + const interpolatedToFbo = createFramebuffer(gl, interpolatedToTex); + let loadingOverlay: SnapshotLoadingOverlay | null = null; + const getLoadingOverlay = (): SnapshotLoadingOverlay | null => { + if (loadingMode !== "internal") return null; + loadingOverlay = loadingOverlay || createSnapshotLoadingOverlay(root, compWidth, compHeight); + return loadingOverlay; + }; + let shaderCacheReady = false; + let prewarming = false; + let sceneMutationSuppressionDepth = 0; + let ignoreSceneMutationsUntil = 0; + const scenePointerEvents = new WeakMap(); - const tickShader = () => { - if (state.active && state.prog) { - const fromTex = textures.get(state.fromId); - const toTex = textures.get(state.toId); - if (fromTex && toTex) { - renderShader( - gl, - quadBuf, - state.prog, - fromTex, - toTex, - state.progress, - accentColors, - compWidth, - compHeight, - ); + const markRuntimeSceneMutation = (): void => { + ignoreSceneMutationsUntil = performance.now() + 120; + }; + + const rememberScenePointerEvents = (scene: HTMLElement): void => { + if (!scenePointerEvents.has(scene)) { + scenePointerEvents.set(scene, scene.style.pointerEvents); + } + }; + + const setScenePlaybackState = (scene: HTMLElement, visible: boolean, opacity: string): void => { + rememberScenePointerEvents(scene); + markRuntimeSceneMutation(); + scene.style.opacity = opacity; + scene.style.visibility = visible ? "visible" : "hidden"; + scene.style.pointerEvents = visible ? (scenePointerEvents.get(scene) ?? "") : "none"; + }; + + const paintScenePairState = ( + fromId: string, + toId: string, + fromOpacity: string, + toOpacity: string, + ): void => { + scenes.forEach((sceneId) => { + const scene = document.getElementById(sceneId); + if (!scene) return; + if (sceneId === fromId) { + setScenePlaybackState(scene, true, fromOpacity); + } else if (sceneId === toId) { + setScenePlaybackState(scene, true, toOpacity); + } else { + setScenePlaybackState(scene, false, "0"); + } + }); + }; + + const disposeCaptureCanvas = (canvas: HTMLCanvasElement): void => { + canvas.width = 0; + canvas.height = 0; + }; + + const captureLiveScene = (scene: HTMLElement): Promise => { + return captureScene(scene, bgColor, compWidth, compHeight, { + forceVisible: true, + scale: previewCaptureScale, + }); + }; + + const waitForPaint = (): Promise => { + return new Promise((resolve) => { + requestAnimationFrame(() => { + requestAnimationFrame(() => resolve()); + }); + }); + }; + + const hasFrameTextures = ( + frame: CachedTransitionFrame | undefined, + ): frame is TexturedTransitionFrame => { + return Boolean(frame?.fromTex && frame.toTex); + }; + + const selectCachedFrameBlend = ( + cache: CachedTransition, + progress: number, + ): CachedTransitionFrameBlend | null => { + if (cache.frames.length === 0) return null; + const position = clampNumber(progress, 0, 1) * (cache.frames.length - 1); + const lowerIndex = Math.floor(position); + const upperIndex = Math.ceil(position); + const a = cache.frames[lowerIndex] ?? cache.frames[cache.frames.length - 1]; + const b = cache.frames[upperIndex] ?? a; + if (!hasFrameTextures(a) || !hasFrameTextures(b)) return null; + return { a, b, mix: position - lowerIndex }; + }; + + const renderTextureBlend = ( + target: WebGLFramebuffer, + texA: WebGLTexture, + texB: WebGLTexture, + mix: number, + ): void => { + gl.bindFramebuffer(gl.FRAMEBUFFER, target); + gl.viewport(0, 0, previewTextureWidth, previewTextureHeight); + gl.useProgram(blendProg); + gl.activeTexture(gl.TEXTURE0); + gl.bindTexture(gl.TEXTURE_2D, texA); + gl.uniform1i(blendLoc.a, 0); + gl.activeTexture(gl.TEXTURE1); + gl.bindTexture(gl.TEXTURE_2D, texB); + gl.uniform1i(blendLoc.b, 1); + gl.uniform1f(blendLoc.mix, mix); + gl.bindBuffer(gl.ARRAY_BUFFER, quadBuf); + gl.enableVertexAttribArray(blendLoc.pos); + gl.vertexAttribPointer(blendLoc.pos, 2, gl.FLOAT, false, 0, 0); + gl.drawArrays(gl.TRIANGLE_STRIP, 0, 4); + gl.bindFramebuffer(gl.FRAMEBUFFER, null); + gl.viewport(0, 0, compWidth, compHeight); + }; + + const preloadTransitionTextures = (cache: CachedTransition): void => { + void ensureTransitionTextures(cache).then((loaded) => { + if (!loaded) return; + const now = tl.time(); + if (now >= cache.time && now < cache.time + cache.duration) { + tickShader(); + } + }); + }; + + const resolveSettledSceneIndex = (currentTime: number): number => { + let index = 0; + for (let i = 0; i < cachedTransitions.length; i += 1) { + const cache = cachedTransitions[i]; + if (cache && currentTime >= cache.time + cache.duration) { + index = i + 1; } } + return clampNumber(index, 0, scenes.length - 1); + }; + + const paintSettledSceneState = (currentTime: number): void => { + const visibleIndex = resolveSettledSceneIndex(currentTime); + scenes.forEach((sceneId, index) => { + const scene = document.getElementById(sceneId); + if (!scene) return; + const visible = index === visibleIndex; + setScenePlaybackState(scene, visible, visible ? "1" : "0"); + }); + }; + + const applyFallbackTransition = (cache: CachedTransition, progress: number): void => { + const fromScene = document.getElementById(cache.fromId); + const toScene = document.getElementById(cache.toId); + if (!fromScene || !toScene) return; + const eased = progress * progress * (3 - 2 * progress); + canvasEl.style.display = "none"; + paintScenePairState(cache.fromId, cache.toId, String(1 - eased), String(eased)); + }; + + const tickShader = () => { + if (prewarming) { + return; + } + + const currentTime = tl.time(); + const upcoming = cachedTransitions.find((cache) => { + return ( + !cache.fallback && + cache.ready && + !cache.dirty && + !cache.textureReady && + currentTime >= cache.time - TEXTURE_PRELOAD_LOOKAHEAD_SECONDS && + currentTime < cache.time + cache.duration + ); + }); + if (upcoming) { + preloadTransitionTextures(upcoming); + } + + const activeIndex = cachedTransitions.findIndex((cache) => { + return currentTime >= cache.time && currentTime < cache.time + cache.duration; + }); + if (activeIndex < 0) { + state.active = false; + state.transitionIndex = -1; + canvasEl.style.display = "none"; + paintSettledSceneState(currentTime); + return; + } + + const cache = cachedTransitions[activeIndex]; + if (!cache || cache.dirty || !cache.ready) { + canvasEl.style.display = "none"; + return; + } + if (cache.fallback) { + state.active = true; + state.transitionIndex = activeIndex; + state.prog = null; + state.progress = clampNumber((currentTime - cache.time) / cache.duration, 0, 1); + applyFallbackTransition(cache, state.progress); + return; + } + if (!cache.textureReady) { + preloadTransitionTextures(cache); + canvasEl.style.display = "none"; + return; + } + + state.active = true; + state.transitionIndex = activeIndex; + state.prog = cache.prog; + state.progress = clampNumber((currentTime - cache.time) / cache.duration, 0, 1); + markTextureAccess(cache); + + const frame = selectCachedFrameBlend(cache, state.progress); + if (!frame) { + canvasEl.style.display = "none"; + return; + } + paintScenePairState(cache.fromId, cache.toId, "1", "1"); + renderTextureBlend(interpolatedFromFbo, frame.a.fromTex, frame.b.fromTex, frame.mix); + renderTextureBlend(interpolatedToFbo, frame.a.toTex, frame.b.toTex, frame.mix); + + canvasEl.style.display = "block"; + renderShader( + gl, + quadBuf, + state.prog, + interpolatedFromTex, + interpolatedToTex, + state.progress, + accentColors, + compWidth, + compHeight, + ); }; let tl: GsapTimeline; @@ -255,11 +1181,110 @@ export function init(config: HyperShaderConfig): GsapTimeline { tl = gsap.timeline({ paused: true, onUpdate: tickShader }); } + const originalPlay = tl.play.bind(tl) as (...args: unknown[]) => GsapTimeline; + const originalPause = tl.pause.bind(tl) as (...args: unknown[]) => GsapTimeline; + const originalTime = tl.time.bind(tl) as (...args: unknown[]) => GsapTimeline | number; + const originalSeek = + typeof tl.seek === "function" + ? (tl.seek.bind(tl) as (...args: unknown[]) => GsapTimeline) + : null; + const readActualTimelineTime = (): number => { + const value = originalTime(); + return typeof value === "number" && Number.isFinite(value) ? value : 0; + }; + let publicTimelineTime = readActualTimelineTime(); + const updatePublicTimelineTime = (value: unknown): void => { + if (typeof value === "number" && Number.isFinite(value)) { + publicTimelineTime = value; + } + }; + const getPlaybackRequestTime = (args: unknown[], fallback: number): number => { + const requestedTime = args[0]; + return typeof requestedTime === "number" && Number.isFinite(requestedTime) + ? requestedTime + : fallback; + }; + const setActualTimelineTime = (time: number, suppressEvents: boolean): GsapTimeline => { + const result = originalTime(time, suppressEvents); + return typeof result === "number" ? tl : result; + }; + const suppressSceneMutationTracking = (fn: () => T): T => { + sceneMutationSuppressionDepth += 1; + try { + return fn(); + } finally { + sceneMutationSuppressionDepth -= 1; + ignoreSceneMutationsUntil = performance.now() + 120; + } + }; + let pendingPlay = false; + let pendingPlayArgs: unknown[] = []; + let cancelResumeAfterPrewarm = false; + tl.play = ((...args: unknown[]) => { + updatePublicTimelineTime(args[0]); + const requestedTime = getPlaybackRequestTime(args, publicTimelineTime); + if (!areAllCachesReady() || !arePlaybackTexturesReady(requestedTime)) { + cancelResumeAfterPrewarm = false; + pendingPlay = true; + pendingPlayArgs = args; + void ensureTransitionCachesReady(); + return tl; + } + const result = suppressSceneMutationTracking(() => originalPlay(...args)); + publicTimelineTime = readActualTimelineTime(); + return result; + }) as GsapTimeline["play"]; + tl.pause = ((...args: unknown[]) => { + pendingPlay = false; + pendingPlayArgs = []; + updatePublicTimelineTime(args[0]); + if (prewarming) { + cancelResumeAfterPrewarm = true; + return tl; + } + const result = suppressSceneMutationTracking(() => originalPause(...args)); + publicTimelineTime = readActualTimelineTime(); + if (args.length > 0) { + tickShader(); + } + return result; + }) as GsapTimeline["pause"]; + tl.time = ((...args: unknown[]) => { + if (args.length === 0) { + if (!prewarming) { + publicTimelineTime = readActualTimelineTime(); + } + return publicTimelineTime; + } + updatePublicTimelineTime(args[0]); + if (prewarming) { + return tl; + } + const result = suppressSceneMutationTracking(() => originalTime(...args)); + if (!prewarming) { + publicTimelineTime = readActualTimelineTime(); + } + tickShader(); + return result; + }) as GsapTimeline["time"]; + if (originalSeek) { + tl.seek = ((...args: unknown[]) => { + updatePublicTimelineTime(args[0]); + if (prewarming) { + return tl; + } + const result = suppressSceneMutationTracking(() => originalSeek(...args)); + if (!prewarming) { + publicTimelineTime = readActualTimelineTime(); + } + tickShader(); + return result; + }) as NonNullable; + } + initCapture(); glCanvas.style.display = "none"; - const canvasEl = glCanvas; - for (let i = 0; i < transitions.length; i++) { const t = transitions[i]; const fromId = scenes[i]; @@ -272,85 +1297,46 @@ export function init(config: HyperShaderConfig): GsapTimeline { const dur = t.duration ?? DEFAULT_DURATION; const ease = t.ease ?? DEFAULT_EASE; const T = t.time; + const cacheIndex = cachedTransitions.length; + cachedTransitions.push({ + index: cacheIndex, + time: T, + duration: dur, + fromId, + toId, + prog, + frames: [], + cacheKey: "", + dirty: true, + ready: false, + fallback: false, + persisted: false, + textureReady: false, + texturePromise: null, + textureGeneration: 0, + textureAccess: 0, + }); - // Pause timeline during async capture to prevent the progress tween - // from running ahead. Resume once textures are uploaded. tl.call( () => { - const fromScene = document.getElementById(fromId); - const toScene = document.getElementById(toId); - if (!fromScene || !toScene) return; + suppressSceneMutationTracking(() => { + const fromScene = document.getElementById(fromId); + const toScene = document.getElementById(toId); + if (!fromScene || !toScene) return; - const wasPlaying = !tl.paused(); - if (wasPlaying) tl.pause(); - - captureScene(fromScene, bgColor, compWidth, compHeight) - .then((fromCanvas) => { - const fromTex = textures.get(fromId); - if (fromTex) uploadTexture(gl, fromTex, fromCanvas); - return captureIncomingScene(toScene, bgColor, compWidth, compHeight); - }) - .then((toCanvas) => { - const toTex = textures.get(toId); - if (toTex) uploadTexture(gl, toTex, toCanvas); - - // Guard: only apply transition-state DOM changes if the playhead - // is STILL inside this transition's [T, T+dur] window. Without - // this, a seek that crosses multiple transitions launches several - // async captures in parallel; each resolves ~80-200ms later and - // unconditionally calls querySelectorAll(".scene").opacity = "0" - // + canvas.display = "block" + state.active = true. The last one - // to resolve wins, so after seeking past a transition, state gets - // stuck pointing at the wrong transition and every scene is - // hidden — manifesting as the "scrub blanks until the next scene - // begins" bug. Checking tl.time() against the transition window - // keeps async capture completions from corrupting state the - // end-callback (at T+dur) or the next transition's start-callback - // has already set correctly. - const nowTime = tl.time(); - const inWindow = nowTime >= T && nowTime < T + dur; - if (inWindow) { - document.querySelectorAll(".scene").forEach((s) => { - s.style.opacity = "0"; - }); - canvasEl.style.display = "block"; - state.prog = prog; - state.fromId = fromId; - state.toId = toId; - state.progress = 0; - state.active = true; - } - - if (wasPlaying) tl.play(); - }) - .catch((e) => { - // Graceful fallback for unavoidable capture failures. The most - // common cause is Safari's stricter canvas-taint rules combined - // with SVG-filter-based background images (e.g. inline - // `` grain data URLs): html2canvas returns a - // tainted canvas, then `gl.texImage2D` throws SecurityError - // with no framework opt-out (WebGL spec). In Chrome this path - // rarely fires, but when it does (CORS-less cross-origin - // images, iframe sandbox restrictions, etc.) the old hard-cut - // was jarring. A CSS crossfade is strictly better UX. - console.warn("[HyperShader] Capture failed, CSS crossfade fallback:", e); - const nowTime = tl.time(); - const inWindow = nowTime >= T && nowTime < T + dur; - if (inWindow) { - const fromEl = document.getElementById(fromId); - const toEl = document.getElementById(toId); - if (fromEl && toEl) { - gsap.to(fromEl, { opacity: 0, duration: dur, ease }); - gsap.fromTo(toEl, { opacity: 0 }, { opacity: 1, duration: dur, ease }); - } else { - document.querySelectorAll(".scene").forEach((s) => { - s.style.opacity = "0"; - }); - if (toEl) toEl.style.opacity = "1"; - } - } - if (wasPlaying) tl.play(); - }); + state.prog = prog; + state.transitionIndex = cacheIndex; + state.progress = 0; + state.active = true; + const cache = cachedTransitions[cacheIndex]; + if (cache?.fallback) { + applyFallbackTransition(cache, 0); + return; + } + canvasEl.style.display = + !prewarming && cache?.ready && !cache.dirty && cache.textureReady ? "block" : "none"; + paintScenePairState(fromId, toId, "1", "1"); + }); }, null, T, @@ -372,16 +1358,794 @@ export function init(config: HyperShaderConfig): GsapTimeline { tl.call( () => { - state.active = false; - canvasEl.style.display = "none"; - const scene = document.getElementById(toId); - if (scene) scene.style.opacity = "1"; + suppressSceneMutationTracking(() => { + state.active = false; + state.transitionIndex = -1; + canvasEl.style.display = "none"; + paintSettledSceneState(T + dur); + }); }, null, T + dur, ); } + type ShaderReadyState = { + ready: boolean; + progress: number; + total: number; + currentTransition?: number; + transitionTotal?: number; + transitionFrame?: number; + transitionFrames?: number; + phase?: SnapshotLoadingStatus["phase"]; + dirtyTransitions: number; + captureScale: number; + textureWidth: number; + textureHeight: number; + fps: number; + loading: boolean; + error?: string; + }; + + const sampleCountForCache = (cache: CachedTransition): number => { + return Math.max(2, Math.ceil(cache.duration * previewCaptureFps) + 1); + }; + + const areAllCachesReady = (): boolean => { + return cachedTransitions.every((cache) => cache.ready && !cache.dirty); + }; + + const getPlaybackTextureWindow = (currentTime: number): CachedTransition[] => { + const selected: CachedTransition[] = []; + const selectedIndexes = new Set(); + const addIfNeeded = (cache: CachedTransition): void => { + if (selectedIndexes.has(cache.index)) return; + if (cache.fallback || cache.dirty || !cache.ready) return; + selected.push(cache); + selectedIndexes.add(cache.index); + }; + + for (const cache of cachedTransitions) { + if (currentTime >= cache.time && currentTime < cache.time + cache.duration) { + addIfNeeded(cache); + } + } + + const nextUpcoming = cachedTransitions.find((cache) => cache.time >= currentTime); + if (nextUpcoming) { + addIfNeeded(nextUpcoming); + } + + for (const cache of cachedTransitions) { + if ( + cache.time >= currentTime && + cache.time - currentTime <= TEXTURE_PRELOAD_LOOKAHEAD_SECONDS + ) { + addIfNeeded(cache); + } + } + return selected.slice(0, MAX_TEXTURED_TRANSITIONS); + }; + + const arePlaybackTexturesReady = (currentTime: number): boolean => { + return getPlaybackTextureWindow(currentTime).every((cache) => cache.textureReady); + }; + + let textureAccessCounter = 0; + + const disposeTransitionTextures = (cache: CachedTransition): void => { + cache.textureGeneration += 1; + for (const frame of cache.frames) { + if (frame.fromTex) { + gl.deleteTexture(frame.fromTex); + frame.fromTex = null; + } + if (frame.toTex) { + gl.deleteTexture(frame.toTex); + frame.toTex = null; + } + } + cache.textureReady = false; + }; + + const disposeCachedTransition = (cache: CachedTransition): void => { + disposeTransitionTextures(cache); + cache.texturePromise = null; + cache.frames = []; + cache.ready = false; + cache.fallback = false; + cache.persisted = false; + cache.textureReady = false; + cache.lastError = undefined; + }; + + const markTextureAccess = (cache: CachedTransition): void => { + textureAccessCounter += 1; + cache.textureAccess = textureAccessCounter; + }; + + const enforceTextureBudget = (keep: CachedTransition): void => { + const loaded = cachedTransitions + .filter((cache) => cache !== keep && cache.textureReady) + .sort((a, b) => a.textureAccess - b.textureAccess); + while (loaded.length >= MAX_TEXTURED_TRANSITIONS) { + const evict = loaded.shift(); + if (!evict) break; + disposeTransitionTextures(evict); + } + }; + + const buildTransitionCacheKey = (cache: CachedTransition, sampleCount: number): string => { + const source = [ + SNAPSHOT_CACHE_SCHEMA, + compId, + cache.index, + cache.fromId, + getSceneSignature(cache.fromId), + cache.toId, + getSceneSignature(cache.toId), + transitions[cache.index]?.shader || "unknown", + cache.time, + cache.duration, + sampleCount, + previewCaptureFps, + previewCaptureScale, + previewTextureWidth, + previewTextureHeight, + compWidth, + compHeight, + ].join("|"); + return `${compId}:${cache.index}:${stableHash(source)}`; + }; + + const setShaderReadyState = (status: Partial) => { + const hfWin = window as unknown as { + __hf?: { + shaderTransitions?: Record; + }; + }; + hfWin.__hf = hfWin.__hf || {}; + hfWin.__hf.shaderTransitions = hfWin.__hf.shaderTransitions || {}; + const current = hfWin.__hf.shaderTransitions[compId] || { + ready: false, + progress: 0, + total: 0, + dirtyTransitions: cachedTransitions.length, + captureScale: previewCaptureScale, + textureWidth: previewTextureWidth, + textureHeight: previewTextureHeight, + fps: previewCaptureFps, + loading: true, + }; + const next = { ...current, ...status }; + next.dirtyTransitions = cachedTransitions.filter((cache) => cache.dirty || !cache.ready).length; + hfWin.__hf.shaderTransitions[compId] = next; + window.parent?.postMessage( + { + source: "hf-preview", + type: "shader-transition-state", + compositionId: compId, + state: next, + }, + "*", + ); + if (next.loading) { + const overlay = getLoadingOverlay(); + overlay?.show(); + overlay?.update({ + progress: next.progress, + total: next.total, + currentTransition: next.currentTransition, + transitionTotal: next.transitionTotal, + transitionFrame: next.transitionFrame, + transitionFrames: next.transitionFrames, + phase: next.phase, + }); + } + if (next.ready) { + loadingOverlay?.hide(); + } else if (!next.loading) { + loadingOverlay?.hide(); + } + }; + + const shouldIgnoreSceneMutation = (): boolean => { + return ( + prewarming || + sceneMutationSuppressionDepth > 0 || + performance.now() < ignoreSceneMutationsUntil || + !tl.paused() + ); + }; + + const markScenesDirty = (sceneIds: Set): void => { + if (sceneIds.size === 0) return; + let changed = false; + for (const cache of cachedTransitions) { + if (!sceneIds.has(cache.fromId) && !sceneIds.has(cache.toId)) continue; + disposeCachedTransition(cache); + cache.dirty = true; + cache.cacheKey = ""; + changed = true; + } + if (!changed) return; + shaderCacheReady = areAllCachesReady(); + setShaderReadyState({ + ready: shaderCacheReady, + progress: 0, + total: 0, + currentTransition: undefined, + transitionTotal: undefined, + transitionFrame: undefined, + transitionFrames: undefined, + phase: undefined, + loading: false, + }); + window.dispatchEvent( + new CustomEvent("hyperShader:dirty", { + detail: { compositionId: compId, scenes: Array.from(sceneIds) }, + }), + ); + }; + + const observeSceneEdits = (): MutationObserver[] => { + const sceneElements = scenes + .map((id) => document.getElementById(id)) + .filter((el): el is HTMLElement => el instanceof HTMLElement); + const observers: MutationObserver[] = []; + for (const scene of sceneElements) { + const observer = new MutationObserver((mutations) => { + if (shouldIgnoreSceneMutation()) return; + const affected = new Set(); + for (const mutation of mutations) { + const target = + mutation.target instanceof Element ? mutation.target : mutation.target.parentElement; + if (!target) continue; + for (const candidate of sceneElements) { + if (candidate === target || candidate.contains(target)) { + affected.add(candidate.id); + } + } + } + markScenesDirty(affected); + }); + observer.observe(scene, { + attributes: true, + characterData: true, + childList: true, + subtree: true, + }); + observers.push(observer); + } + + const styleObserver = new MutationObserver(() => { + if (shouldIgnoreSceneMutation()) return; + markScenesDirty(new Set(scenes)); + }); + styleObserver.observe(document.head, { + attributes: true, + characterData: true, + childList: true, + subtree: true, + }); + observers.push(styleObserver); + return observers; + }; + + const hydrateTransitionCache = async ( + cache: CachedTransition, + sampleCount: number, + onProgress: (transitionFrame: number) => void, + ): Promise => { + const hydratedFrames: CachedTransitionFrame[] = []; + for (let sampleIndex = 0; sampleIndex < sampleCount; sampleIndex += 1) { + const [fromEntry, toEntry] = await Promise.all([ + getSnapshotEntry(makeSnapshotKey(cache.cacheKey, sampleIndex, "from")), + getSnapshotEntry(makeSnapshotKey(cache.cacheKey, sampleIndex, "to")), + ]); + if (!fromEntry || !toEntry) { + for (const frame of hydratedFrames) { + gl.deleteTexture(frame.fromTex); + gl.deleteTexture(frame.toTex); + } + return false; + } + + hydratedFrames.push({ + sampleIndex, + fromBlob: fromEntry.blob, + toBlob: toEntry.blob, + fromTex: null, + toTex: null, + }); + onProgress(sampleIndex + 1); + } + cache.frames = hydratedFrames; + cache.ready = true; + cache.dirty = false; + cache.fallback = false; + cache.persisted = true; + cache.textureReady = false; + return true; + }; + + const ensureTransitionTextures = (cache: CachedTransition): Promise => { + if (cache.fallback || cache.dirty || !cache.ready) return Promise.resolve(false); + if (cache.textureReady) { + markTextureAccess(cache); + return Promise.resolve(true); + } + if (cache.texturePromise) return cache.texturePromise; + + const generation = cache.textureGeneration; + const frames = cache.frames; + const uploadedTextures: WebGLTexture[] = []; + const isStaleTextureJob = (): boolean => { + return cache.textureGeneration !== generation || cache.dirty || cache.frames !== frames; + }; + const disposeUploadedTextures = (): void => { + const deleted = new Set(); + for (const tex of uploadedTextures) { + if (!deleted.has(tex)) { + gl.deleteTexture(tex); + deleted.add(tex); + } + } + for (const frame of frames) { + if (frame.fromTex && !deleted.has(frame.fromTex)) { + gl.deleteTexture(frame.fromTex); + deleted.add(frame.fromTex); + } + if (frame.toTex && !deleted.has(frame.toTex)) { + gl.deleteTexture(frame.toTex); + deleted.add(frame.toTex); + } + frame.fromTex = null; + frame.toTex = null; + } + }; + const getFrameBlob = async ( + frame: CachedTransitionFrame, + side: "from" | "to", + ): Promise => { + const cached = side === "from" ? frame.fromBlob : frame.toBlob; + if (cached) return cached; + const entry = await getSnapshotEntry( + makeSnapshotKey(cache.cacheKey, frame.sampleIndex, side), + ); + if (entry?.blob) return entry.blob; + throw new Error("[HyperShader] Cached transition snapshot blob is unavailable"); + }; + + let texturePromise = Promise.resolve(false); + texturePromise = (async () => { + for (const frame of frames) { + if (isStaleTextureJob()) { + disposeUploadedTextures(); + return false; + } + if (!frame.fromTex) { + const fromBlob = await getFrameBlob(frame, "from"); + if (isStaleTextureJob()) { + disposeUploadedTextures(); + return false; + } + const source = await blobToTextureSource(fromBlob); + const tex = createTexture(gl); + uploadedTextures.push(tex); + try { + uploadTextureSource(gl, tex, source); + if (isStaleTextureJob()) { + disposeUploadedTextures(); + return false; + } + frame.fromTex = tex; + } finally { + closeTextureSource(source); + } + } + if (!frame.toTex) { + const toBlob = await getFrameBlob(frame, "to"); + if (isStaleTextureJob()) { + disposeUploadedTextures(); + return false; + } + const source = await blobToTextureSource(toBlob); + const tex = createTexture(gl); + uploadedTextures.push(tex); + try { + uploadTextureSource(gl, tex, source); + if (isStaleTextureJob()) { + disposeUploadedTextures(); + return false; + } + frame.toTex = tex; + } finally { + closeTextureSource(source); + } + } + if (cache.persisted) { + frame.fromBlob = null; + frame.toBlob = null; + } + } + if (isStaleTextureJob()) { + disposeUploadedTextures(); + return false; + } + cache.textureReady = frames.every((frame) => Boolean(frame.fromTex && frame.toTex)); + if (cache.textureReady) { + markTextureAccess(cache); + enforceTextureBudget(cache); + } + return cache.textureReady; + })() + .catch((e) => { + disposeUploadedTextures(); + if (isStaleTextureJob()) { + return false; + } + disposeTransitionTextures(cache); + cache.fallback = true; + cache.ready = true; + cache.dirty = false; + cache.lastError = e instanceof Error ? e.message : String(e); + setShaderReadyState({ + ready: areAllCachesReady(), + loading: false, + error: cache.lastError, + }); + return false; + }) + .finally(() => { + if (cache.texturePromise === texturePromise) { + cache.texturePromise = null; + } + }); + + cache.texturePromise = texturePromise; + return texturePromise; + }; + + const ensurePlaybackTextureWindow = async (currentTime: number): Promise => { + for (const cache of getPlaybackTextureWindow(currentTime)) { + await ensureTransitionTextures(cache); + } + }; + + const persistSnapshot = async ( + cache: CachedTransition, + sampleIndex: number, + side: "from" | "to", + blob: Blob | null, + ): Promise => { + if (!blob) return false; + return putSnapshotEntry({ + key: makeSnapshotKey(cache.cacheKey, sampleIndex, side), + blob, + width: previewTextureWidth, + height: previewTextureHeight, + updatedAt: cache.index * 1_000_000 + sampleIndex * 2 + (side === "to" ? 1 : 0), + }); + }; + + const captureTransitionCache = async ( + cache: CachedTransition, + sampleCount: number, + onProgress: (transitionFrame: number) => void, + ): Promise => { + disposeCachedTransition(cache); + let allPersisted = true; + for (let sampleIndex = 0; sampleIndex < sampleCount; sampleIndex += 1) { + const progress = sampleIndex / (sampleCount - 1); + suppressSceneMutationTracking(() => { + originalTime(cache.time + cache.duration * progress, false); + }); + await waitForPaint(); + + const fromScene = document.getElementById(cache.fromId); + const toScene = document.getElementById(cache.toId); + if (!fromScene || !toScene) continue; + + const [fromCanvas, toCanvas] = await Promise.all([ + captureLiveScene(fromScene), + captureLiveScene(toScene), + ]); + const [fromBlob, toBlob] = await Promise.all([ + canvasToPngBlob(fromCanvas), + canvasToPngBlob(toCanvas), + ]); + disposeCaptureCanvas(fromCanvas); + disposeCaptureCanvas(toCanvas); + if (!fromBlob || !toBlob) { + throw new Error("[HyperShader] Failed to encode transition snapshot"); + } + + cache.frames.push({ sampleIndex, fromBlob, toBlob, fromTex: null, toTex: null }); + const persisted = await Promise.all([ + persistSnapshot(cache, sampleIndex, "from", fromBlob), + persistSnapshot(cache, sampleIndex, "to", toBlob), + ]); + if (!persisted.every(Boolean)) { + allPersisted = false; + cache.lastError = "[HyperShader] Failed to persist one or more transition snapshots"; + } + onProgress(sampleIndex + 1); + } + cache.ready = true; + cache.dirty = false; + cache.fallback = false; + cache.persisted = allPersisted; + cache.textureReady = false; + }; + + let transitionCachePromise: Promise | null = null; + + const ensureTransitionCachesReady = (): Promise => { + if (transitionCachePromise) return transitionCachePromise; + + transitionCachePromise = (async () => { + const work = cachedTransitions.filter((cache) => cache.dirty || !cache.ready); + const workItems = work.map((cache) => ({ + cache, + sampleCount: sampleCountForCache(cache), + })); + for (const item of workItems) { + item.cache.cacheKey = buildTransitionCacheKey(item.cache, item.sampleCount); + } + const total = workItems.reduce((sum, item) => sum + item.sampleCount, 0); + const firstItem = workItems[0]; + let showLoadingOverlay = false; + setShaderReadyState({ + ready: false, + progress: 0, + total, + currentTransition: firstItem ? 1 : undefined, + transitionTotal: workItems.length || undefined, + transitionFrame: firstItem ? 0 : undefined, + transitionFrames: firstItem?.sampleCount, + phase: firstItem ? "cached" : undefined, + loading: showLoadingOverlay, + }); + + if (work.length === 0) { + shaderCacheReady = true; + setShaderReadyState({ + ready: true, + progress: 0, + total: 0, + currentTransition: undefined, + transitionTotal: undefined, + transitionFrame: undefined, + transitionFrames: undefined, + phase: undefined, + loading: false, + }); + if (pendingPlay) { + const resumeArgs = pendingPlayArgs; + const resumeTime = getPlaybackRequestTime(resumeArgs, publicTimelineTime); + await ensurePlaybackTextureWindow(resumeTime); + if (pendingPlay) { + pendingPlay = false; + pendingPlayArgs = []; + cancelResumeAfterPrewarm = false; + suppressSceneMutationTracking(() => originalPlay(...resumeArgs)); + publicTimelineTime = readActualTimelineTime(); + } + } + transitionCachePromise = null; + return; + } + + publicTimelineTime = readActualTimelineTime(); + const wasPaused = tl.paused(); + const originalSceneStyles: SceneStyleState[] = scenes.map((id) => { + const scene = document.getElementById(id); + return { + scene, + opacity: scene?.style.opacity ?? "", + visibility: scene?.style.visibility ?? "", + pointerEvents: scene?.style.pointerEvents ?? "", + }; + }); + + prewarming = true; + canvasEl.style.display = "none"; + originalPause(); + + let completed = 0; + try { + for (let workIndex = 0; workIndex < workItems.length; workIndex += 1) { + const item = workItems[workIndex]; + if (!item) continue; + const { cache, sampleCount } = item; + const currentTransition = workIndex + 1; + const completedBeforeTransition = completed; + disposeCachedTransition(cache); + try { + setShaderReadyState({ + ready: false, + progress: completedBeforeTransition, + total, + currentTransition, + transitionTotal: workItems.length, + transitionFrame: 0, + transitionFrames: sampleCount, + phase: "cached", + loading: showLoadingOverlay, + }); + const hydrated = await hydrateTransitionCache(cache, sampleCount, (transitionFrame) => { + completed = completedBeforeTransition + transitionFrame; + setShaderReadyState({ + ready: false, + progress: completed, + total, + currentTransition, + transitionTotal: workItems.length, + transitionFrame, + transitionFrames: sampleCount, + phase: "cached", + loading: showLoadingOverlay, + }); + }); + if (!hydrated) { + completed = completedBeforeTransition; + showLoadingOverlay = true; + setShaderReadyState({ + ready: false, + progress: completed, + total, + currentTransition, + transitionTotal: workItems.length, + transitionFrame: 0, + transitionFrames: sampleCount, + phase: "capturing", + loading: true, + }); + await captureTransitionCache(cache, sampleCount, (transitionFrame) => { + completed = completedBeforeTransition + transitionFrame; + setShaderReadyState({ + ready: false, + progress: completed, + total, + currentTransition, + transitionTotal: workItems.length, + transitionFrame, + transitionFrames: sampleCount, + phase: "capturing", + loading: true, + }); + }); + if (!cache.persisted && cache.lastError) { + setShaderReadyState({ + ready: false, + loading: showLoadingOverlay, + error: cache.lastError, + }); + } + } + } catch (e) { + completed = completedBeforeTransition + sampleCount; + cache.fallback = true; + cache.ready = true; + cache.dirty = false; + cache.lastError = e instanceof Error ? e.message : String(e); + console.warn("[HyperShader] Transition capture failed; using CSS fallback:", e); + setShaderReadyState({ + ready: false, + progress: completed, + total, + currentTransition, + transitionTotal: workItems.length, + transitionFrame: sampleCount, + transitionFrames: sampleCount, + phase: "finalizing", + loading: showLoadingOverlay, + error: cache.lastError, + }); + } + completed = completedBeforeTransition + sampleCount; + } + void pruneSnapshotCache( + compId, + new Set(cachedTransitions.map((cache) => cache.cacheKey).filter(Boolean)), + ); + shaderCacheReady = areAllCachesReady(); + const finalItem = workItems[workItems.length - 1]; + setShaderReadyState({ + ready: shaderCacheReady, + progress: completed, + total, + currentTransition: workItems.length, + transitionTotal: workItems.length, + transitionFrame: finalItem?.sampleCount, + transitionFrames: finalItem?.sampleCount, + phase: "finalizing", + loading: false, + }); + } catch (e) { + console.warn("[HyperShader] Pre-capture failed, keeping DOM fallback visible:", e); + shaderCacheReady = areAllCachesReady(); + setShaderReadyState({ + ready: shaderCacheReady, + progress: completed, + total, + phase: "finalizing", + loading: false, + error: e instanceof Error ? e.message : String(e), + }); + } finally { + const restoreTimelineTime = publicTimelineTime; + let shouldResume = (pendingPlay || !wasPaused) && !cancelResumeAfterPrewarm; + let resumeArgs = pendingPlay ? pendingPlayArgs : []; + let resumeTime = getPlaybackRequestTime(resumeArgs, restoreTimelineTime); + let hydratedTextureTime: number | null = null; + if (shouldResume) { + await ensurePlaybackTextureWindow(resumeTime); + hydratedTextureTime = resumeTime; + } + shouldResume = (pendingPlay || !wasPaused) && !cancelResumeAfterPrewarm; + resumeArgs = pendingPlay ? pendingPlayArgs : []; + resumeTime = getPlaybackRequestTime(resumeArgs, restoreTimelineTime); + if (shouldResume && resumeTime !== hydratedTextureTime) { + await ensurePlaybackTextureWindow(resumeTime); + } + prewarming = false; + state.active = false; + state.transitionIndex = -1; + canvasEl.style.display = "none"; + suppressSceneMutationTracking(() => { + setActualTimelineTime(restoreTimelineTime, false); + }); + publicTimelineTime = restoreTimelineTime; + for (const item of originalSceneStyles) { + if (!item.scene) continue; + item.scene.style.opacity = item.opacity; + item.scene.style.visibility = item.visibility; + item.scene.style.pointerEvents = item.pointerEvents; + } + tickShader(); + window.dispatchEvent( + new CustomEvent("hyperShader:ready", { + detail: { compositionId: compId, progress: completed, total }, + }), + ); + transitionCachePromise = null; + if (shouldResume) { + pendingPlay = false; + pendingPlayArgs = []; + cancelResumeAfterPrewarm = false; + suppressSceneMutationTracking(() => originalPlay(...resumeArgs)); + publicTimelineTime = readActualTimelineTime(); + } else { + pendingPlay = false; + pendingPlayArgs = []; + cancelResumeAfterPrewarm = false; + originalPause(); + } + } + })(); + + return transitionCachePromise; + }; + + const sceneEditObservers = observeSceneEdits(); + window.addEventListener( + "beforeunload", + () => { + for (const observer of sceneEditObservers) { + observer.disconnect(); + } + }, + { once: true }, + ); + + const prewarmPromise = Promise.resolve().then(() => ensureTransitionCachesReady()); + const hfWin = window as unknown as { __hf?: { shaderTransitionsReady?: Promise } }; + hfWin.__hf = hfWin.__hf || {}; + hfWin.__hf.shaderTransitionsReady = prewarmPromise; + registerTimeline(compId, tl, config.timeline); return tl; } diff --git a/packages/shader-transitions/src/webgl.ts b/packages/shader-transitions/src/webgl.ts index 3025229ae..a36ffe51f 100644 --- a/packages/shader-transitions/src/webgl.ts +++ b/packages/shader-transitions/src/webgl.ts @@ -36,13 +36,14 @@ function compileShader(gl: WebGLRenderingContext, src: string, type: number): We return s; } -export function createProgram(gl: WebGLRenderingContext, fragSrc: string): WebGLProgram { - if (!cachedVertexShader) { - cachedVertexShader = compileShader(gl, vertSrc, gl.VERTEX_SHADER); - } +function linkProgram( + gl: WebGLRenderingContext, + vertexShader: WebGLShader, + fragSrc: string, +): WebGLProgram { const p = gl.createProgram(); if (!p) throw new Error("[HyperShader] Failed to create program"); - gl.attachShader(p, cachedVertexShader); + gl.attachShader(p, vertexShader); gl.attachShader(p, compileShader(gl, fragSrc, gl.FRAGMENT_SHADER)); gl.linkProgram(p); if (!gl.getProgramParameter(p, gl.LINK_STATUS)) { @@ -51,6 +52,21 @@ export function createProgram(gl: WebGLRenderingContext, fragSrc: string): WebGL return p; } +export function createProgram(gl: WebGLRenderingContext, fragSrc: string): WebGLProgram { + if (!cachedVertexShader) { + cachedVertexShader = compileShader(gl, vertSrc, gl.VERTEX_SHADER); + } + return linkProgram(gl, cachedVertexShader, fragSrc); +} + +export function createProgramWithVertex( + gl: WebGLRenderingContext, + vertexSrc: string, + fragSrc: string, +): WebGLProgram { + return linkProgram(gl, compileShader(gl, vertexSrc, gl.VERTEX_SHADER), fragSrc); +} + export interface AccentColors { accent: [number, number, number]; dark: [number, number, number]; @@ -136,8 +152,16 @@ export function uploadTexture( tex: WebGLTexture, canvas: HTMLCanvasElement, ): void { - gl.bindTexture(gl.TEXTURE_2D, tex); - gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, canvas); + uploadTextureSource(gl, tex, canvas); canvas.width = 0; canvas.height = 0; } + +export function uploadTextureSource( + gl: WebGLRenderingContext, + tex: WebGLTexture, + source: TexImageSource, +): void { + gl.bindTexture(gl.TEXTURE_2D, tex); + gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, source); +} diff --git a/packages/studio/src/components/ui/HyperframesLoader.tsx b/packages/studio/src/components/ui/HyperframesLoader.tsx new file mode 100644 index 000000000..5a31e1a48 --- /dev/null +++ b/packages/studio/src/components/ui/HyperframesLoader.tsx @@ -0,0 +1,104 @@ +export interface HyperframesLoaderProps { + /** Status text shown below the mark. */ + title: string; + /** Optional secondary detail line. */ + detail?: string; + /** Optional monospace third line for IDs, counts, or percentages. */ + mono?: string; + /** Pixel size of the mark itself; status text scales independently. */ + size?: number; + /** Optional normalized progress value from 0 to 1. */ + progress?: number; +} + +export function HyperframesLoader({ + title, + detail, + mono, + size = 64, + progress, +}: HyperframesLoaderProps) { + const boundedProgress = + typeof progress === "number" && Number.isFinite(progress) + ? Math.min(1, Math.max(0, progress)) + : undefined; + const markFrameSize = Math.round(size * 1.16); + + return ( +
+
+ +
+
{title}
+ {detail &&
{detail}
} + {boundedProgress !== undefined && ( + + ); +} + +export function StatusFrame(props: HyperframesLoaderProps) { + return ( +
+ +
+ ); +} diff --git a/packages/studio/src/components/ui/index.ts b/packages/studio/src/components/ui/index.ts index 3f5812e0c..9623c7362 100644 --- a/packages/studio/src/components/ui/index.ts +++ b/packages/studio/src/components/ui/index.ts @@ -1,2 +1,4 @@ // Minimal UI primitives for studio canvas components export { Button, IconButton } from "./Button"; +export { HyperframesLoader, StatusFrame } from "./HyperframesLoader"; +export type { HyperframesLoaderProps } from "./HyperframesLoader"; diff --git a/packages/studio/src/player/components/Player.tsx b/packages/studio/src/player/components/Player.tsx index 519536d31..cd90ee07c 100644 --- a/packages/studio/src/player/components/Player.tsx +++ b/packages/studio/src/player/components/Player.tsx @@ -1,6 +1,7 @@ -import { forwardRef, useRef, useState } from "react"; +import { forwardRef, useEffect, useRef, useState } from "react"; import { isLottieAnimationLoaded } from "@hyperframes/core/runtime/lottie-readiness"; import { useMountEffect } from "../../hooks/useMountEffect"; +import { HyperframesLoader } from "../../components/ui"; // NOTE: importing "@hyperframes/player" registers a class extending HTMLElement // at module load, which throws under SSR. Defer the import to the mount effect // so it only runs in the browser. @@ -16,6 +17,19 @@ interface HyperframesPlayerElement extends HTMLElement { iframeElement: HTMLIFrameElement; } +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null; +} + +function getShaderTransitionLoading(event: Event): boolean | null { + if (!(event instanceof CustomEvent)) return null; + const detail: unknown = event.detail; + if (!isRecord(detail)) return null; + const state = detail.state; + if (!isRecord(state)) return null; + return state.loading === true && state.ready !== true; +} + // Assets are considered ready when every `