mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-09-03 04:38:33 +00:00
feat(producer): true alpha output for webm, mov, and png-sequence
Extends RenderConfig.format with "png-sequence" and patches two correctness
gaps so the existing "webm" / "mov" values actually preserve the alpha
channel end-to-end.
Engine fixes:
- screenshotService.pageScreenshotCapture: drop optimizeForSpeed for PNG
captures. The fast path uses an alpha-unaware codec that crushes real
alpha values; kept for opaque jpeg captures where it is harmless.
- frameCapture: replace the inline setDefaultBackgroundColorOverride
block (which fired pre-navigation and was reset by page.goto) with a
proper initTransparentBackground() call inside initializeSession,
after the window.__hf readiness poll. This also injects the
html/body/[data-composition-id]{background:transparent !important}
stylesheet so compositions with custom body / #root backgrounds do not
defeat the override. Wired into both screenshot-mode and beginframe-mode
branches.
Producer:
- RenderConfig.format extended to "mp4" | "webm" | "mov" | "png-sequence"
with full JSDoc.
- Streaming encode is bypassed for png-sequence (frames go straight to
disk). FORMAT_EXT extended.
- New Stage-5 png-sequence branch: mkdir outputPath, copy captured PNGs as
frame_NNNNNN.png, copy audio.aac sidecar when audio is present.
- Stage-6 mux/faststart and the debug copy are wrapped in !isPngSequence.
- README.md: new "Transparent Video Output" section.
Tests:
- New fixture tests/transparency-regression/ tagged "transparency".
- New tsx script src/transparency-test.ts asserts pixel-level alpha for
webm + png-sequence outputs. Wired as "test:transparency".
- Default "test" / "test:update" scripts pass --exclude-tags transparency
so the golden-MP4 harness ignores the new fixture.
Verified locally on macOS arm64: typecheck clean across engine + producer,
producer renderOrchestrator vitest 10/10, transparency-test passes for
both webm and png-sequence with end-to-end pixel assertions.
This commit is contained in:
Youssef Toufik
@@ -20,7 +20,12 @@ import {
|
||||
resolveHeadlessShellPath,
|
||||
type CaptureMode,
|
||||
} from "./browserManager.js";
|
||||
import { beginFrameCapture, getCdpSession, pageScreenshotCapture } from "./screenshotService.js";
|
||||
import {
|
||||
beginFrameCapture,
|
||||
getCdpSession,
|
||||
pageScreenshotCapture,
|
||||
initTransparentBackground,
|
||||
} from "./screenshotService.js";
|
||||
import { DEFAULT_CONFIG, type EngineConfig } from "../config.js";
|
||||
import type {
|
||||
CaptureOptions,
|
||||
@@ -78,7 +83,11 @@ export async function createCaptureSession(
|
||||
): Promise<CaptureSession> {
|
||||
if (!existsSync(outputDir)) mkdirSync(outputDir, { recursive: true });
|
||||
|
||||
// Determine capture mode before building args — BeginFrame flags only apply on Linux
|
||||
// Determine capture mode before building args — BeginFrame flags only apply on Linux.
|
||||
// BeginFrame's compositor does not preserve alpha; callers that pass
|
||||
// `options.format === "png"` for transparent capture should also set
|
||||
// `config.forceScreenshot = true` (the producer's renderOrchestrator does this
|
||||
// automatically when `RenderConfig.format` is an alpha-capable value).
|
||||
const headlessShell = resolveHeadlessShellPath(config);
|
||||
const isLinux = process.platform === "linux";
|
||||
const forceScreenshot = config?.forceScreenshot ?? DEFAULT_CONFIG.forceScreenshot;
|
||||
@@ -144,15 +153,12 @@ export async function createCaptureSession(
|
||||
};
|
||||
await page.setViewport(viewport);
|
||||
|
||||
// For PNG capture (used by WebM/transparency), make the page background transparent
|
||||
// so Chrome's screenshot captures alpha channel data. Must use the same CDP session
|
||||
// that the screenshot service uses (getCdpSession caches per page).
|
||||
if (options.format === "png") {
|
||||
const cdp = await getCdpSession(page);
|
||||
await cdp.send("Emulation.setDefaultBackgroundColorOverride", {
|
||||
color: { r: 0, g: 0, b: 0, a: 0 },
|
||||
});
|
||||
}
|
||||
// Transparent-background setup is intentionally NOT done here. Chrome resets
|
||||
// the default-background-color override on navigation, and the
|
||||
// `[data-composition-id]{background:transparent}` stylesheet that
|
||||
// `initTransparentBackground` injects must land in a real `document.head`.
|
||||
// See `initializeSession()` below — it calls `initTransparentBackground` for
|
||||
// PNG captures after `page.goto(...)` and the `window.__hf` readiness poll.
|
||||
|
||||
return {
|
||||
browser,
|
||||
@@ -303,6 +309,17 @@ export async function initializeSession(session: CaptureSession): Promise<void>
|
||||
|
||||
await page.evaluate(`document.fonts?.ready`);
|
||||
|
||||
// For PNG captures, force the page background fully transparent so the
|
||||
// captured screenshots carry a real alpha channel. Must run AFTER
|
||||
// navigation (Chrome resets the override on every goto) and AFTER the
|
||||
// page is loaded (the injected stylesheet needs a real document.head).
|
||||
// The override is overridden by `body { background: ... }` and
|
||||
// `#root { background: ... }` rules — the helper handles that with a
|
||||
// `[data-composition-id]{background:transparent !important}` injection.
|
||||
if (session.options.format === "png") {
|
||||
await initTransparentBackground(session.page);
|
||||
}
|
||||
|
||||
session.isInitialized = true;
|
||||
return;
|
||||
}
|
||||
@@ -388,6 +405,16 @@ export async function initializeSession(session: CaptureSession): Promise<void>
|
||||
// Set base frame time ticks past warmup range
|
||||
session.beginFrameTimeTicks = (warmupTicks + 10) * session.beginFrameIntervalMs;
|
||||
|
||||
// For PNG captures, inject the transparent-background override + stylesheet
|
||||
// (see the screenshot-mode branch above for the rationale). BeginFrame mode
|
||||
// does not actually preserve alpha through its compositor — callers that
|
||||
// need transparent output should set `forceScreenshot: true` so this branch
|
||||
// is bypassed entirely. The call is left here as defense-in-depth for any
|
||||
// future BeginFrame alpha support.
|
||||
if (session.options.format === "png") {
|
||||
await initTransparentBackground(session.page);
|
||||
}
|
||||
|
||||
session.isInitialized = true;
|
||||
}
|
||||
|
||||
|
||||
@@ -119,16 +119,22 @@ export async function beginFrameCapture(
|
||||
/**
|
||||
* Capture a screenshot using standard Page.captureScreenshot CDP call.
|
||||
* Fallback for environments where BeginFrame is unavailable (macOS, Windows).
|
||||
*
|
||||
* For `format: "png"` captures we disable Chrome's `optimizeForSpeed` fast
|
||||
* path. The fast path uses a zero-alpha-aware codec that crushes real alpha
|
||||
* values to 0 or 255 (verified empirically; CDP docs don't document this) —
|
||||
* exactly the same caveat called out on `captureScreenshotWithAlpha` /
|
||||
* `captureAlphaPng`. Keeping the fast path for opaque jpeg captures is fine.
|
||||
*/
|
||||
export async function pageScreenshotCapture(page: Page, options: CaptureOptions): Promise<Buffer> {
|
||||
const client = await getCdpSession(page);
|
||||
const format = options.format === "png" ? "png" : "jpeg";
|
||||
const isPng = options.format === "png";
|
||||
const result = await client.send("Page.captureScreenshot", {
|
||||
format,
|
||||
quality: format === "jpeg" ? (options.quality ?? 80) : undefined,
|
||||
format: isPng ? "png" : "jpeg",
|
||||
quality: isPng ? undefined : (options.quality ?? 80),
|
||||
fromSurface: true,
|
||||
captureBeyondViewport: false,
|
||||
optimizeForSpeed: true,
|
||||
optimizeForSpeed: !isPng,
|
||||
});
|
||||
return Buffer.from(result.data, "base64");
|
||||
}
|
||||
|
||||
+62
-12
@@ -47,22 +47,72 @@ await startServer({ port: 8080 });
|
||||
|
||||
`RenderConfig` controls the render pipeline:
|
||||
|
||||
| Option | Default | Description |
|
||||
| ------------ | ------------ | -------------------------------------------------- |
|
||||
| `inputPath` | — | Path to the HTML composition |
|
||||
| `outputPath` | — | Output video file path |
|
||||
| `width` | 1920 | Frame width in pixels |
|
||||
| `height` | 1080 | Frame height in pixels |
|
||||
| `fps` | 30 | Frames per second (24, 30, or 60) |
|
||||
| `quality` | `"standard"` | Encoder preset (`"draft"`, `"standard"`, `"high"`) |
|
||||
| Option | Default | Description |
|
||||
| ------------ | ------------ | ------------------------------------------------------------------------------------------------------------------------------------ |
|
||||
| `inputPath` | — | Path to the HTML composition |
|
||||
| `outputPath` | — | Output video file path (or directory, for `format: "png-sequence"`) |
|
||||
| `width` | 1920 | Frame width in pixels |
|
||||
| `height` | 1080 | Frame height in pixels |
|
||||
| `fps` | 30 | Frames per second (24, 30, or 60) |
|
||||
| `quality` | `"standard"` | Encoder preset (`"draft"`, `"standard"`, `"high"`) |
|
||||
| `format` | `"mp4"` | Output container — `"mp4"`, `"webm"`, `"mov"`, or `"png-sequence"`. See [Transparent Video Output](#transparent-video-output) below. |
|
||||
|
||||
## Transparent Video Output
|
||||
|
||||
The producer can render HTML compositions to formats that carry a **true alpha channel** — not chroma key. The same composition that renders an opaque MP4 renders a layerable overlay when you set `format`.
|
||||
|
||||
| `format` | Codec / pixel format | Alpha | Audio | Use case |
|
||||
| ----------------- | --------------------------------- | ----------------------- | ------------------- | --------------------------------------------------------------------------------------- |
|
||||
| `"mp4"` (default) | H.264 (yuv420p) or H.265 + HDR10 | No | AAC | Streaming, social, default deliverable |
|
||||
| `"webm"` | VP9 + yuva420p | **True alpha** | Opus | Web playback as overlay (`<video>` over background); supported in Chrome, Edge, Firefox |
|
||||
| `"mov"` | ProRes 4444 + yuva444p10le | **True alpha + 10-bit** | AAC | Editor ingest (Premiere, Final Cut Pro, DaVinci Resolve) |
|
||||
| `"png-sequence"` | Numbered RGBA PNGs in a directory | **Lossless alpha** | Sidecar `audio.aac` | After Effects / Nuke / Fusion, or pipelines that post-process frames before encoding |
|
||||
|
||||
### Example
|
||||
|
||||
```typescript
|
||||
import { createRenderJob, executeRenderJob } from "@hyperframes/producer";
|
||||
|
||||
const job = createRenderJob({
|
||||
inputPath: "./my-composition.html",
|
||||
outputPath: "./output.webm", // or a directory for "png-sequence"
|
||||
width: 1080,
|
||||
height: 1920,
|
||||
fps: 30,
|
||||
format: "webm", // "mp4" | "webm" | "mov" | "png-sequence"
|
||||
});
|
||||
|
||||
await executeRenderJob(job);
|
||||
```
|
||||
|
||||
### What "transparent background" means here
|
||||
|
||||
The producer captures Chrome screenshots with the page background forced transparent (`html, body, [data-composition-id] { background: transparent !important }`) and the CDP default background override set to RGBA 0,0,0,0. The captured PNGs carry a real alpha channel and that channel is preserved end-to-end:
|
||||
|
||||
- VP9 (`webm`) is encoded with `-pix_fmt yuva420p`, `-auto-alt-ref 0`, and `alpha_mode=1` metadata.
|
||||
- ProRes 4444 (`mov`) is encoded with `-pix_fmt yuva444p10le`.
|
||||
- PNG sequences are written without re-encoding (zero-padded `frame_NNNNNN.png`).
|
||||
|
||||
This is not chroma keying. There is no green/blue background to remove and no "key" tolerance to tune — pixels that were transparent in the browser are transparent in the output.
|
||||
|
||||
### Caveats
|
||||
|
||||
- **Linux + alpha forces screenshot capture.** Chrome's BeginFrame compositor (the default deterministic capture path on Linux headless-shell) does not preserve alpha; the orchestrator falls back to `Page.captureScreenshot`, which is slower per frame. macOS and Windows already use screenshot mode by default, so they are unaffected.
|
||||
- **HDR + alpha is not supported.** Setting `hdr: true` together with an alpha-capable format logs a warning and falls back to SDR. Use `format: "mp4"` for HDR10 output.
|
||||
- **`png-sequence` does not produce a single muxed file.** When the composition contains audio elements, an `audio.aac` sidecar is written alongside the PNGs in `outputPath`.
|
||||
- **Safari + WebM alpha is incomplete.** For broad browser playback of an alpha video, ship `format: "mov"` to your editor and re-encode for the codec your distribution target supports.
|
||||
|
||||
### Authoring transparent compositions
|
||||
|
||||
Don't paint a fullscreen background in your HTML. The default body background is overridden to transparent automatically — any `body { background: ... }`, `#root { background: ... }`, or `[data-composition-id] { background: ... }` rule is force-overridden during alpha rendering. Backgrounds on inner elements (cards, scenes, components) are kept.
|
||||
|
||||
## How it works
|
||||
|
||||
1. **Serve** — spins up a local file server for the HTML composition
|
||||
2. **Capture** — opens the page in headless Chrome, seeks frame-by-frame via `HeadlessExperimental.beginFrame`, captures screenshots
|
||||
3. **Encode** — pipes frames through FFmpeg (with GPU encoder detection and chunked concat)
|
||||
4. **Mix** — extracts `<audio>` elements and mixes them into the final video
|
||||
5. **Finalize** — applies faststart for streaming-friendly MP4
|
||||
2. **Capture** — opens the page in headless Chrome, seeks frame-by-frame via `HeadlessExperimental.beginFrame` (or `Page.captureScreenshot` for transparent / non-Linux renders), captures screenshots
|
||||
3. **Encode** — pipes frames through FFmpeg (with GPU encoder detection and chunked concat). Skipped for `format: "png-sequence"`.
|
||||
4. **Mix** — extracts `<audio>` elements and mixes them into the final video. For `png-sequence`, audio is written as an `audio.aac` sidecar.
|
||||
5. **Finalize** — applies faststart for streaming-friendly MP4 (no-op for WebM, MOV, and `png-sequence`)
|
||||
|
||||
## Documentation
|
||||
|
||||
|
||||
@@ -38,8 +38,9 @@
|
||||
"check:runtime-conformance": "tsx src/runtime-conformance.ts",
|
||||
"benchmark": "tsx src/benchmark.ts",
|
||||
"bench:hdr": "tsx src/benchmark.ts --tags hdr",
|
||||
"test": "tsx src/regression-harness.ts",
|
||||
"test:update": "tsx src/regression-harness.ts --update",
|
||||
"test": "tsx src/regression-harness.ts --exclude-tags transparency",
|
||||
"test:update": "tsx src/regression-harness.ts --update --exclude-tags transparency",
|
||||
"test:transparency": "tsx src/transparency-test.ts",
|
||||
"docker:build:test": "docker build -f ../../Dockerfile.test -t hyperframes-producer:test ../..",
|
||||
"docker:test": "docker run --rm --security-opt seccomp=unconfined --shm-size=2g -v ./tests:/app/packages/producer/tests hyperframes-producer:test",
|
||||
"docker:test:update": "docker run --rm --security-opt seccomp=unconfined --shm-size=2g -v ./tests:/app/packages/producer/tests hyperframes-producer:test --update",
|
||||
|
||||
@@ -207,8 +207,40 @@ export type RenderStatus =
|
||||
export interface RenderConfig {
|
||||
fps: 24 | 30 | 60;
|
||||
quality: "draft" | "standard" | "high";
|
||||
/** Output container format. WebM uses VP9+alpha, MOV uses ProRes 4444+alpha for transparency. */
|
||||
format?: "mp4" | "webm" | "mov";
|
||||
/**
|
||||
* Output container format. Defaults to `"mp4"`; existing renders are
|
||||
* unaffected unless this field is set explicitly.
|
||||
*
|
||||
* - `"mp4"`: H.264 (or H.265 + HDR10 when `hdr: true`). Opaque. The
|
||||
* default streaming/social deliverable. Faststart is applied so the
|
||||
* `moov` atom sits at the file start and the file plays from a
|
||||
* partial download.
|
||||
* - `"webm"`: VP9 + `yuva420p` pixel format → **true alpha channel**, no
|
||||
* chroma key. Plays in Chrome, Edge, and Firefox; Safari support for
|
||||
* alpha-WebM is incomplete. Use this when the output should drop
|
||||
* straight into a `<video>` over a colored background on the web.
|
||||
* Audio is muxed as Opus.
|
||||
* - `"mov"`: ProRes 4444 + `yuva444p10le` → **true alpha channel +
|
||||
* 10-bit color**. Sized for editor ingest (Premiere, Final Cut Pro,
|
||||
* DaVinci Resolve), not direct web playback. Audio is muxed as AAC.
|
||||
* - `"png-sequence"`: a directory of zero-padded RGBA PNGs
|
||||
* (`frame_000001.png` …). Lossless alpha, largest on disk, no muxed
|
||||
* audio (an `audio.aac` sidecar is written alongside the PNGs when
|
||||
* the composition has audio elements). Use for After Effects / Nuke
|
||||
* / Fusion ingest, or when frames need post-processing before
|
||||
* encoding. `outputPath` is treated as a directory; it is created if
|
||||
* it doesn't exist.
|
||||
*
|
||||
* Alpha output (`"webm"`, `"mov"`, `"png-sequence"`) automatically
|
||||
* forces screenshot capture (Chrome's BeginFrame compositor does not
|
||||
* preserve alpha on Linux headless-shell) and disables HDR — HDR +
|
||||
* alpha is not a supported combination, a warning is logged and HDR
|
||||
* falls back to SDR. The transparent-background CSS is injected by
|
||||
* the engine's `initTransparentBackground` helper, so authors should
|
||||
* not paint a fullscreen `body` / `#root` background in their
|
||||
* compositions when targeting alpha output.
|
||||
*/
|
||||
format?: "mp4" | "webm" | "mov" | "png-sequence";
|
||||
workers?: number;
|
||||
useGpu?: boolean;
|
||||
debug?: boolean;
|
||||
@@ -997,17 +1029,22 @@ export async function executeRenderJob(
|
||||
};
|
||||
const perfOutputPath = join(workDir, "perf-summary.json");
|
||||
const cfg = { ...(job.config.producerConfig ?? resolveConfig()) };
|
||||
const outputFormat = (job.config.format ?? "mp4") as "mp4" | "webm" | "mov";
|
||||
const outputFormat = (job.config.format ?? "mp4") as "mp4" | "webm" | "mov" | "png-sequence";
|
||||
const isWebm = outputFormat === "webm";
|
||||
const isMov = outputFormat === "mov";
|
||||
const needsAlpha = isWebm || isMov;
|
||||
const isPngSequence = outputFormat === "png-sequence";
|
||||
const needsAlpha = isWebm || isMov || isPngSequence;
|
||||
// Transparency requires screenshot mode — beginFrame doesn't support alpha channel
|
||||
if (needsAlpha) {
|
||||
cfg.forceScreenshot = true;
|
||||
}
|
||||
const enableChunkedEncode = cfg.enableChunkedEncode;
|
||||
const chunkedEncodeSize = cfg.chunkSizeFrames;
|
||||
const enableStreamingEncode = cfg.enableStreamingEncode;
|
||||
// Streaming encode pipes captured frames through ffmpeg's stdin to produce
|
||||
// a single video file. png-sequence has no encoded video output — frames go
|
||||
// straight to disk — so the streaming branch is bypassed regardless of the
|
||||
// engine config flag.
|
||||
const enableStreamingEncode = cfg.enableStreamingEncode && !isPngSequence;
|
||||
|
||||
// Periodic memory sampler — surfaces peak RSS/heap so the benchmark harness
|
||||
// can detect memory regressions (e.g. unbounded image-cache growth) that
|
||||
@@ -1494,7 +1531,8 @@ export async function executeRenderJob(
|
||||
}
|
||||
if (effectiveHdr && outputFormat !== "mp4") {
|
||||
log.warn(
|
||||
`[Render] HDR source detected but format is ${outputFormat} — falling back to SDR. Use --format mp4 for HDR10 output.`,
|
||||
`[Render] HDR source detected but format is "${outputFormat}" — falling back to SDR. ` +
|
||||
`HDR + alpha is not supported. Use --format mp4 for HDR10 output.`,
|
||||
);
|
||||
effectiveHdr = undefined;
|
||||
}
|
||||
@@ -1571,7 +1609,16 @@ export async function executeRenderJob(
|
||||
|
||||
const workerCount = calculateOptimalWorkers(totalFrames, job.config.workers, cfg);
|
||||
|
||||
const FORMAT_EXT: Record<string, string> = { mp4: ".mp4", webm: ".webm", mov: ".mov" };
|
||||
// png-sequence is "no container" — outputPath is treated as a directory and
|
||||
// the encode/mux/faststart stages are skipped entirely. The empty extension
|
||||
// keeps `videoOnlyPath` (which is constructed below) sensible even though
|
||||
// it will not be written.
|
||||
const FORMAT_EXT: Record<string, string> = {
|
||||
mp4: ".mp4",
|
||||
webm: ".webm",
|
||||
mov: ".mov",
|
||||
"png-sequence": "",
|
||||
};
|
||||
const videoExt = FORMAT_EXT[outputFormat] ?? ".mp4";
|
||||
const videoOnlyPath = join(workDir, `video-only${videoExt}`);
|
||||
// Only use the HDR encoder preset when there's HDR content to pass through —
|
||||
@@ -1581,7 +1628,12 @@ export async function executeRenderJob(
|
||||
const nativeHdrIds = new Set([...nativeHdrVideoIds, ...nativeHdrImageIds]);
|
||||
const hasHdrContent = effectiveHdr && nativeHdrIds.size > 0;
|
||||
const encoderHdr = hasHdrContent ? effectiveHdr : undefined;
|
||||
const preset = getEncoderPreset(job.config.quality, outputFormat, encoderHdr);
|
||||
// png-sequence has no encoder, but the rest of the orchestrator still
|
||||
// reads `preset.quality` for `effectiveQuality` and `preset.codec` for
|
||||
// unrelated bookkeeping. Fall back to the mp4 preset shape — its values
|
||||
// are never written to ffmpeg in the png-sequence path.
|
||||
const presetFormat: "mp4" | "webm" | "mov" = isPngSequence ? "mp4" : outputFormat;
|
||||
const preset = getEncoderPreset(job.config.quality, presetFormat, encoderHdr);
|
||||
|
||||
// CLI overrides (--crf, --video-bitrate) flow through job.config and must
|
||||
// win over the preset-derived defaults. The CLI enforces mutual exclusivity
|
||||
@@ -2535,47 +2587,80 @@ export async function executeRenderJob(
|
||||
|
||||
perfStages.captureMs = Date.now() - stage4Start;
|
||||
|
||||
// ── Stage 5: Encode ─────────────────────────────────────────────────
|
||||
const stage5Start = Date.now();
|
||||
updateJobStatus(job, "encoding", "Encoding video", 75, onProgress);
|
||||
|
||||
const frameExt = needsAlpha ? "png" : "jpg";
|
||||
const framePattern = `frame_%06d.${frameExt}`;
|
||||
const encoderOpts = {
|
||||
fps: job.config.fps,
|
||||
width,
|
||||
height,
|
||||
codec: preset.codec,
|
||||
preset: preset.preset,
|
||||
quality: effectiveQuality,
|
||||
bitrate: effectiveBitrate,
|
||||
pixelFormat: preset.pixelFormat,
|
||||
useGpu: job.config.useGpu,
|
||||
hdr: preset.hdr,
|
||||
};
|
||||
const encodeResult = enableChunkedEncode
|
||||
? await encodeFramesChunkedConcat(
|
||||
framesDir,
|
||||
framePattern,
|
||||
videoOnlyPath,
|
||||
encoderOpts,
|
||||
chunkedEncodeSize,
|
||||
abortSignal,
|
||||
)
|
||||
: await encodeFramesFromDir(
|
||||
framesDir,
|
||||
framePattern,
|
||||
videoOnlyPath,
|
||||
encoderOpts,
|
||||
abortSignal,
|
||||
if (isPngSequence) {
|
||||
// ── Stage 5 (png-sequence): copy captured PNGs to outputDir ──────
|
||||
// No encoder, no mux, no faststart — captured frames already carry
|
||||
// alpha and are the deliverable. We rename to `frame_NNNNNN.png`
|
||||
// (zero-padded) so consumers (After Effects, Nuke, Fusion, ffmpeg
|
||||
// image2 demuxer) can globbed-import without surprises.
|
||||
const stage5Start = Date.now();
|
||||
updateJobStatus(job, "encoding", "Writing PNG sequence", 75, onProgress);
|
||||
if (!existsSync(outputPath)) mkdirSync(outputPath, { recursive: true });
|
||||
const captured = readdirSync(framesDir)
|
||||
.filter((name) => name.endsWith(".png"))
|
||||
.sort();
|
||||
if (captured.length === 0) {
|
||||
throw new Error(
|
||||
`[Render] png-sequence output requested but no PNGs were captured to ${framesDir}`,
|
||||
);
|
||||
assertNotAborted();
|
||||
}
|
||||
captured.forEach((name, i) => {
|
||||
const dst = join(outputPath, `frame_${String(i + 1).padStart(6, "0")}.png`);
|
||||
copyFileSync(join(framesDir, name), dst);
|
||||
});
|
||||
if (hasAudio && existsSync(audioOutputPath)) {
|
||||
// Sidecar audio for callers that need to re-mux later. png-sequence
|
||||
// has no container of its own, so this is the only place audio
|
||||
// can land alongside the frames.
|
||||
copyFileSync(audioOutputPath, join(outputPath, "audio.aac"));
|
||||
log.info(
|
||||
`[Render] png-sequence: audio.aac sidecar written to ${outputPath}/audio.aac`,
|
||||
);
|
||||
}
|
||||
perfStages.encodeMs = Date.now() - stage5Start;
|
||||
} else {
|
||||
// ── Stage 5: Encode ───────────────────────────────────────────────
|
||||
const stage5Start = Date.now();
|
||||
updateJobStatus(job, "encoding", "Encoding video", 75, onProgress);
|
||||
|
||||
if (!encodeResult.success) {
|
||||
throw new Error(`Encoding failed: ${encodeResult.error}`);
|
||||
const frameExt = needsAlpha ? "png" : "jpg";
|
||||
const framePattern = `frame_%06d.${frameExt}`;
|
||||
const encoderOpts = {
|
||||
fps: job.config.fps,
|
||||
width,
|
||||
height,
|
||||
codec: preset.codec,
|
||||
preset: preset.preset,
|
||||
quality: effectiveQuality,
|
||||
bitrate: effectiveBitrate,
|
||||
pixelFormat: preset.pixelFormat,
|
||||
useGpu: job.config.useGpu,
|
||||
hdr: preset.hdr,
|
||||
};
|
||||
const encodeResult = enableChunkedEncode
|
||||
? await encodeFramesChunkedConcat(
|
||||
framesDir,
|
||||
framePattern,
|
||||
videoOnlyPath,
|
||||
encoderOpts,
|
||||
chunkedEncodeSize,
|
||||
abortSignal,
|
||||
)
|
||||
: await encodeFramesFromDir(
|
||||
framesDir,
|
||||
framePattern,
|
||||
videoOnlyPath,
|
||||
encoderOpts,
|
||||
abortSignal,
|
||||
);
|
||||
assertNotAborted();
|
||||
|
||||
if (!encodeResult.success) {
|
||||
throw new Error(`Encoding failed: ${encodeResult.error}`);
|
||||
}
|
||||
|
||||
perfStages.encodeMs = Date.now() - stage5Start;
|
||||
}
|
||||
|
||||
perfStages.encodeMs = Date.now() - stage5Start;
|
||||
}
|
||||
} finally {
|
||||
// Defensive cleanup: if the streaming encoder branch threw before
|
||||
@@ -2609,30 +2694,34 @@ export async function executeRenderJob(
|
||||
fileServer = null;
|
||||
|
||||
// ── Stage 6: Assemble ───────────────────────────────────────────────
|
||||
const stage6Start = Date.now();
|
||||
updateJobStatus(job, "assembling", "Assembling final video", 90, onProgress);
|
||||
// Skipped for png-sequence — there is no encoded video to mux/faststart.
|
||||
// The frames were copied directly to outputPath in Stage 5.
|
||||
if (!isPngSequence) {
|
||||
const stage6Start = Date.now();
|
||||
updateJobStatus(job, "assembling", "Assembling final video", 90, onProgress);
|
||||
|
||||
if (hasAudio) {
|
||||
const muxResult = await muxVideoWithAudio(
|
||||
videoOnlyPath,
|
||||
audioOutputPath,
|
||||
outputPath,
|
||||
abortSignal,
|
||||
);
|
||||
assertNotAborted();
|
||||
if (!muxResult.success) {
|
||||
throw new Error(`Audio muxing failed: ${muxResult.error}`);
|
||||
}
|
||||
} else {
|
||||
const faststartResult = await applyFaststart(videoOnlyPath, outputPath, abortSignal);
|
||||
assertNotAborted();
|
||||
if (!faststartResult.success) {
|
||||
throw new Error(`Faststart failed: ${faststartResult.error}`);
|
||||
if (hasAudio) {
|
||||
const muxResult = await muxVideoWithAudio(
|
||||
videoOnlyPath,
|
||||
audioOutputPath,
|
||||
outputPath,
|
||||
abortSignal,
|
||||
);
|
||||
assertNotAborted();
|
||||
if (!muxResult.success) {
|
||||
throw new Error(`Audio muxing failed: ${muxResult.error}`);
|
||||
}
|
||||
} else {
|
||||
const faststartResult = await applyFaststart(videoOnlyPath, outputPath, abortSignal);
|
||||
assertNotAborted();
|
||||
if (!faststartResult.success) {
|
||||
throw new Error(`Faststart failed: ${faststartResult.error}`);
|
||||
}
|
||||
}
|
||||
|
||||
perfStages.assembleMs = Date.now() - stage6Start;
|
||||
}
|
||||
|
||||
perfStages.assembleMs = Date.now() - stage6Start;
|
||||
|
||||
// ── Complete ─────────────────────────────────────────────────────────
|
||||
job.outputPath = outputPath;
|
||||
updateJobStatus(job, "complete", "Render complete", 100, onProgress);
|
||||
@@ -2681,8 +2770,11 @@ export async function executeRenderJob(
|
||||
|
||||
// ── Cleanup ─────────────────────────────────────────────────────────
|
||||
if (job.config.debug) {
|
||||
// Copy output MP4 into debug dir for easy access
|
||||
if (existsSync(outputPath)) {
|
||||
// Copy output MP4 (or single-file alpha output) into the debug dir for
|
||||
// easy access. Skipped for png-sequence: outputPath is a directory, not
|
||||
// a single file — the captured frames already live in `framesDir` under
|
||||
// workDir during a debug run anyway.
|
||||
if (!isPngSequence && existsSync(outputPath)) {
|
||||
const debugOutput = join(workDir, `output${videoExt}`);
|
||||
copyFileSync(outputPath, debugOutput);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,215 @@
|
||||
/**
|
||||
* Transparency Regression Test
|
||||
*
|
||||
* Exercises the alpha-output pipelines (webm + png-sequence) end-to-end
|
||||
* against `tests/transparency-regression/`. Asserts that:
|
||||
*
|
||||
* 1. Pixels that were transparent in the browser stay transparent in the
|
||||
* output (alpha = 0).
|
||||
* 2. Pixels covered by the opaque red `.card` element stay fully opaque
|
||||
* (alpha = 255) and keep their red color.
|
||||
*
|
||||
* This is intentionally NOT wired into `regression-harness.ts` — the harness
|
||||
* compares each fixture against a golden MP4, but transparency requires a
|
||||
* different validation strategy (pixel inspection of the alpha channel). Run
|
||||
* this script via `bun run --filter @hyperframes/producer test:transparency`
|
||||
* or directly via `tsx src/transparency-test.ts` from this package.
|
||||
*/
|
||||
|
||||
import { strict as assert } from "node:assert";
|
||||
import { existsSync, mkdirSync, readdirSync, readFileSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { dirname, join, resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
import { decodePng, runFfmpeg } from "@hyperframes/engine";
|
||||
import { createRenderJob, executeRenderJob } from "./services/renderOrchestrator.js";
|
||||
|
||||
const moduleDir = dirname(fileURLToPath(import.meta.url));
|
||||
const FIXTURE_DIR = resolve(moduleDir, "../tests/transparency-regression");
|
||||
const FIXTURE_SRC = join(FIXTURE_DIR, "src");
|
||||
|
||||
const WIDTH = 200;
|
||||
const HEIGHT = 200;
|
||||
const FPS = 30;
|
||||
const TRANSPARENT_X = 10; // expected fully transparent
|
||||
const TRANSPARENT_Y = 10;
|
||||
const OPAQUE_X = 100; // inside the 50–150 red card
|
||||
const OPAQUE_Y = 100;
|
||||
|
||||
function pixelOffset(x: number, y: number, width: number): number {
|
||||
return (y * width + x) * 4;
|
||||
}
|
||||
|
||||
function assertAlphaPixel(
|
||||
png: { data: Uint8Array; width: number; height: number },
|
||||
x: number,
|
||||
y: number,
|
||||
expectAlpha: "transparent" | "opaque-red",
|
||||
label: string,
|
||||
): void {
|
||||
assert.equal(png.width, WIDTH, `${label}: width mismatch`);
|
||||
assert.equal(png.height, HEIGHT, `${label}: height mismatch`);
|
||||
const off = pixelOffset(x, y, png.width);
|
||||
const r = png.data[off + 0];
|
||||
const g = png.data[off + 1];
|
||||
const b = png.data[off + 2];
|
||||
const a = png.data[off + 3];
|
||||
if (expectAlpha === "transparent") {
|
||||
assert.equal(
|
||||
a,
|
||||
0,
|
||||
`${label}: pixel (${x},${y}) expected fully transparent (alpha=0), got rgba(${r},${g},${b},${a})`,
|
||||
);
|
||||
} else {
|
||||
assert.equal(
|
||||
a,
|
||||
255,
|
||||
`${label}: pixel (${x},${y}) expected fully opaque (alpha=255), got rgba(${r},${g},${b},${a})`,
|
||||
);
|
||||
assert.ok(
|
||||
typeof r === "number" && r >= 240,
|
||||
`${label}: pixel (${x},${y}) expected red >= 240, got rgba(${r},${g},${b},${a})`,
|
||||
);
|
||||
assert.ok(
|
||||
typeof g === "number" && g <= 30,
|
||||
`${label}: pixel (${x},${y}) expected green <= 30, got rgba(${r},${g},${b},${a})`,
|
||||
);
|
||||
assert.ok(
|
||||
typeof b === "number" && b <= 30,
|
||||
`${label}: pixel (${x},${y}) expected blue <= 30, got rgba(${r},${g},${b},${a})`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async function extractFirstFrameFromWebm(webmPath: string, outPng: string): Promise<void> {
|
||||
// VP9 alpha is encoded as a separate intra-frame stream inside the WebM,
|
||||
// and ffmpeg's default decoder path silently discards it. Forcing the
|
||||
// libvpx-vp9 decoder via `-c:v libvpx-vp9` BEFORE `-i` is what engages
|
||||
// the alpha-aware decode — without it the captured transparent pixels
|
||||
// come out opaque even when the file was correctly encoded as yuva420p.
|
||||
// `-update 1` permits writing a single PNG (no `%d` pattern in the path)
|
||||
// and silences the otherwise-noisy ffmpeg warning.
|
||||
const result = await runFfmpeg(
|
||||
[
|
||||
"-y",
|
||||
"-c:v",
|
||||
"libvpx-vp9",
|
||||
"-i",
|
||||
webmPath,
|
||||
"-frames:v",
|
||||
"1",
|
||||
"-pix_fmt",
|
||||
"rgba",
|
||||
"-update",
|
||||
"1",
|
||||
outPng,
|
||||
],
|
||||
{ timeout: 60_000 },
|
||||
);
|
||||
if (!result.success) {
|
||||
throw new Error(
|
||||
`ffmpeg failed extracting frame 0 from ${webmPath}: ${result.stderr.slice(-400)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async function runWebmCheck(workRoot: string): Promise<void> {
|
||||
console.log("\n[webm] rendering transparency-regression …");
|
||||
const outDir = join(workRoot, "webm");
|
||||
mkdirSync(outDir, { recursive: true });
|
||||
const outPath = join(outDir, "out.webm");
|
||||
|
||||
const job = createRenderJob({
|
||||
fps: FPS,
|
||||
quality: "draft",
|
||||
format: "webm",
|
||||
});
|
||||
|
||||
await executeRenderJob(job, FIXTURE_SRC, outPath);
|
||||
assert.equal(job.status, "complete", `webm render did not complete: status=${job.status}`);
|
||||
assert.ok(existsSync(outPath), `webm output not written to ${outPath}`);
|
||||
const size = (await import("node:fs")).statSync(outPath).size;
|
||||
assert.ok(size > 0, `webm output ${outPath} is empty`);
|
||||
console.log(`[webm] rendered ${outPath} (${size} bytes)`);
|
||||
|
||||
const framePng = join(outDir, "frame-0.png");
|
||||
await extractFirstFrameFromWebm(outPath, framePng);
|
||||
const decoded = decodePng(readFileSync(framePng));
|
||||
assertAlphaPixel(decoded, TRANSPARENT_X, TRANSPARENT_Y, "transparent", "webm");
|
||||
assertAlphaPixel(decoded, OPAQUE_X, OPAQUE_Y, "opaque-red", "webm");
|
||||
console.log("[webm] PASS — transparent + opaque-red pixels verified");
|
||||
}
|
||||
|
||||
async function runPngSequenceCheck(workRoot: string): Promise<void> {
|
||||
console.log("\n[png-sequence] rendering transparency-regression …");
|
||||
const outDir = join(workRoot, "pngs");
|
||||
// executeRenderJob mkdirs outputPath itself; deliberately leave it absent.
|
||||
|
||||
const job = createRenderJob({
|
||||
fps: FPS,
|
||||
quality: "draft",
|
||||
format: "png-sequence",
|
||||
});
|
||||
|
||||
await executeRenderJob(job, FIXTURE_SRC, outDir);
|
||||
assert.equal(
|
||||
job.status,
|
||||
"complete",
|
||||
`png-sequence render did not complete: status=${job.status}`,
|
||||
);
|
||||
assert.ok(existsSync(outDir), `png-sequence output dir missing: ${outDir}`);
|
||||
|
||||
const frames = readdirSync(outDir)
|
||||
.filter((name) => name.startsWith("frame_") && name.endsWith(".png"))
|
||||
.sort();
|
||||
assert.equal(
|
||||
frames.length,
|
||||
FPS, // 1 second at 30fps = 30 frames
|
||||
`png-sequence expected ${FPS} frames, got ${frames.length}: ${frames.join(",")}`,
|
||||
);
|
||||
assert.equal(frames[0], "frame_000001.png", "first frame should be frame_000001.png");
|
||||
assert.equal(
|
||||
frames[frames.length - 1],
|
||||
`frame_${String(FPS).padStart(6, "0")}.png`,
|
||||
`last frame should be frame_${String(FPS).padStart(6, "0")}.png`,
|
||||
);
|
||||
console.log(`[png-sequence] wrote ${frames.length} frames to ${outDir}`);
|
||||
|
||||
const firstFrame = frames[0];
|
||||
if (!firstFrame) throw new Error("png-sequence: first frame missing");
|
||||
const decoded = decodePng(readFileSync(join(outDir, firstFrame)));
|
||||
assertAlphaPixel(decoded, TRANSPARENT_X, TRANSPARENT_Y, "transparent", "png-sequence");
|
||||
assertAlphaPixel(decoded, OPAQUE_X, OPAQUE_Y, "opaque-red", "png-sequence");
|
||||
console.log("[png-sequence] PASS — transparent + opaque-red pixels verified");
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
if (!existsSync(FIXTURE_SRC)) {
|
||||
throw new Error(`Fixture missing: ${FIXTURE_SRC}`);
|
||||
}
|
||||
const workRoot = join(tmpdir(), `hf-transparency-${process.pid}-${Date.now()}`);
|
||||
mkdirSync(workRoot, { recursive: true });
|
||||
const keepWork = process.env.KEEP_TEMP === "1";
|
||||
console.log(`work dir: ${workRoot}${keepWork ? " (KEEP_TEMP=1)" : ""}`);
|
||||
|
||||
try {
|
||||
await runWebmCheck(workRoot);
|
||||
await runPngSequenceCheck(workRoot);
|
||||
console.log("\nAll transparency assertions passed.");
|
||||
} finally {
|
||||
if (!keepWork) {
|
||||
try {
|
||||
rmSync(workRoot, { recursive: true, force: true });
|
||||
} catch {
|
||||
// best-effort cleanup
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error("\nTransparency regression test FAILED:");
|
||||
console.error(err);
|
||||
process.exitCode = 1;
|
||||
});
|
||||
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"name": "Transparency Regression",
|
||||
"description": "Asserts that webm + png-sequence outputs preserve a real alpha channel end-to-end. Exercised by `tsx src/transparency-test.ts`, NOT by the standard regression harness (which compares against a golden MP4).",
|
||||
"tags": ["transparency", "alpha", "smoke"],
|
||||
"renderConfig": {
|
||||
"fps": 30
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
<!doctype html>
|
||||
<html>
|
||||
<head>
|
||||
<style>
|
||||
html,
|
||||
body {
|
||||
margin: 0;
|
||||
padding: 0;
|
||||
}
|
||||
/*
|
||||
Intentionally NO body / #root background — the alpha pipeline forces
|
||||
the page background fully transparent. We DO set a background on
|
||||
.card to verify that *inner* element backgrounds survive the
|
||||
force-transparent override.
|
||||
*/
|
||||
.card {
|
||||
position: absolute;
|
||||
left: 50px;
|
||||
top: 50px;
|
||||
width: 100px;
|
||||
height: 100px;
|
||||
background: rgb(255, 0, 0);
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body
|
||||
data-composition-id="transparency-regression"
|
||||
data-duration="1"
|
||||
data-width="200"
|
||||
data-height="200"
|
||||
>
|
||||
<div class="card"></div>
|
||||
<script>
|
||||
// Minimal seek protocol — the composition is static.
|
||||
window.__hf = {
|
||||
duration: 1,
|
||||
seek: () => {},
|
||||
};
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
Reference in New Issue
Block a user