mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-09-05 00:56:23 +00:00
feat(engine): add HDR video output pipeline (#265)
## Summary Adds the ability to render HDR video output (H.265 10-bit, BT.2020) from HyperFrames compositions. When the renderer detects HDR source video, it automatically switches to the HDR output pipeline — no flags needed. ## What it does - **Auto-detection** — Probes each video source with `ffprobe`. If any has bt2020/PQ/HLG color metadata, the output switches to H.265 10-bit with correct color tags. SDR-only compositions are unaffected (H.264, bt709). - **HLG pass-through** — Native HLG pixels from FFmpeg extraction are piped directly to the encoder without conversion. This avoids brightness loss from HLG→linear→PQ conversion (which requires an OOTF system gamma we can't reliably apply). - **Encoder HDR support** — Both chunk and streaming encoders accept HDR presets: `libx265`, `yuv420p10le`, BT.2020 color primaries, `hvc1` codec tag (required for Apple playback). - **WebGPU HDR capture (gated)** — A complete WebGPU float16 readback pipeline is implemented and tested but gated behind headed Chrome (headless doesn't expose WebGPU). Ready for future use with WebGPU canvas content. - **HDR utilities** — `detectTransfer()` (PQ vs HLG), `getHdrEncoderColorParams()`, `analyzeCompositionHdr()`. 15 unit tests. ## Key design decisions | Decision | Why | |----------|-----| | No `--hdr` flag | SDR content encoded as HDR causes orange shift in browsers. Auto-detect eliminates this. | | HLG pass-through (not HLG→PQ) | Conversion loses brightness without OOTF. Pass-through matches source exactly. | | `hvc1` codec tag | Apple QuickTime requires `hvc1` (not `hev1`) for HEVC playback. | | 1-hour streaming timeout | HDR capture at ~6fps needs more time than the default 10-minute FFmpeg timeout. | ## Files changed | File | What changed | |------|-------------| | `packages/engine/src/utils/hdr.ts` | **NEW** — HDR detection, transfer types, encoder params (15 tests) | | `packages/engine/src/services/hdrCapture.ts` | **NEW** — WebGPU readback, HLG conversion, PQ encode | | `packages/engine/src/services/streamingEncoder.ts` | HDR presets, raw rgb48le input, color tags | | `packages/engine/src/services/chunkEncoder.ts` | HDR presets, conditional color tags | | `packages/producer/src/services/renderOrchestrator.ts` | Auto-detection loop, HDR pass-through capture path | ## How to test Render a composition with an HDR video source. The output should be H.265 10-bit with HDR metadata visible in `ffprobe` (bt2020, arib-std-b67 or smpte2084). Plays correctly in QuickTime and on HDR displays. ## Stack position **2 of 6** — Stacked on #258 (SDR/HDR normalization). Provides the encoder infrastructure that phases 1-5 build on. 🤖 Generated with [Claude Code](https://claude.com/claude-code)
This commit is contained in:
Vance Ingalls
@@ -0,0 +1,159 @@
|
||||
import { describe, it, expect } from "vitest";
|
||||
import { float16ToPqRgb } from "./hdrCapture.js";
|
||||
|
||||
// IEEE 754 half-precision (float16) bit patterns used to feed
|
||||
// `float16ToPqRgb`. Encoding rule: sign(1) | exp(5) | frac(10).
|
||||
const F16_ZERO = 0x0000; // +0.0
|
||||
const F16_HALF = 0x3800; // +0.5 (exp=14, frac=0 → 2^-1)
|
||||
const F16_ONE = 0x3c00; // +1.0 (exp=15, frac=0 → 2^0 — SDR white)
|
||||
// PQ caps at 10000 nits and SDR_NITS = 203, so the linear input must exceed
|
||||
// ~58x SDR white before linearToPQ(L) clips at 1.0. 1024 is well above that.
|
||||
const F16_OVERBRIGHT = 0x6400; // +1024.0 (exp=25, frac=0 → 2^10)
|
||||
|
||||
function makeFloat16Frame(
|
||||
width: number,
|
||||
height: number,
|
||||
pixel: { r: number; g: number; b: number; a: number },
|
||||
bytesPerRow: number = width * 8,
|
||||
): Buffer {
|
||||
// Row-padded layout matches WebGPU readback: bytesPerRow ≥ width * 8 (4
|
||||
// channels × 2 bytes), with garbage bytes after each row's pixel data.
|
||||
const buf = Buffer.alloc(height * bytesPerRow);
|
||||
for (let y = 0; y < height; y++) {
|
||||
for (let x = 0; x < width; x++) {
|
||||
const idx = y * bytesPerRow + x * 8;
|
||||
buf.writeUInt16LE(pixel.r, idx);
|
||||
buf.writeUInt16LE(pixel.g, idx + 2);
|
||||
buf.writeUInt16LE(pixel.b, idx + 4);
|
||||
buf.writeUInt16LE(pixel.a, idx + 6);
|
||||
}
|
||||
}
|
||||
return buf;
|
||||
}
|
||||
|
||||
describe("float16ToPqRgb", () => {
|
||||
it("returns a buffer of width * height * 6 bytes (rgb48le)", () => {
|
||||
const frame = makeFloat16Frame(4, 3, { r: 0, g: 0, b: 0, a: 0 });
|
||||
const out = float16ToPqRgb(frame, 32, 4, 3);
|
||||
expect(out.length).toBe(4 * 3 * 6);
|
||||
});
|
||||
|
||||
it("encodes float16 black to PQ zero (linearToPQ(0) ≈ 0 after uint16 quantization)", () => {
|
||||
const frame = makeFloat16Frame(2, 2, {
|
||||
r: F16_ZERO,
|
||||
g: F16_ZERO,
|
||||
b: F16_ZERO,
|
||||
a: F16_ZERO,
|
||||
});
|
||||
const out = float16ToPqRgb(frame, 16, 2, 2);
|
||||
for (let i = 0; i < out.length; i += 2) {
|
||||
expect(out.readUInt16LE(i)).toBe(0);
|
||||
}
|
||||
});
|
||||
|
||||
it("clamps overbright float16 input to PQ 65535 (linearToPQ(>>1.0) → 1.0)", () => {
|
||||
// ~1024 linear is well past the 58x-SDR PQ saturation point; output caps
|
||||
// at 1.0 → 65535 in uint16.
|
||||
const frame = makeFloat16Frame(2, 2, {
|
||||
r: F16_OVERBRIGHT,
|
||||
g: F16_OVERBRIGHT,
|
||||
b: F16_OVERBRIGHT,
|
||||
a: F16_ZERO,
|
||||
});
|
||||
const out = float16ToPqRgb(frame, 16, 2, 2);
|
||||
for (let pixel = 0; pixel < 4; pixel++) {
|
||||
const dst = pixel * 6;
|
||||
expect(out.readUInt16LE(dst)).toBe(65535);
|
||||
expect(out.readUInt16LE(dst + 2)).toBe(65535);
|
||||
expect(out.readUInt16LE(dst + 4)).toBe(65535);
|
||||
}
|
||||
});
|
||||
|
||||
it("preserves channel ordering R, G, B (alpha is discarded)", () => {
|
||||
// Distinct float16 values per channel verify the function doesn't
|
||||
// mix them up. Alpha is set high but should not appear in the output.
|
||||
const frame = makeFloat16Frame(1, 1, {
|
||||
r: F16_ONE,
|
||||
g: F16_HALF,
|
||||
b: F16_ZERO,
|
||||
a: F16_ONE,
|
||||
});
|
||||
const out = float16ToPqRgb(frame, 8, 1, 1);
|
||||
const r = out.readUInt16LE(0);
|
||||
const g = out.readUInt16LE(2);
|
||||
const b = out.readUInt16LE(4);
|
||||
expect(r).toBeGreaterThan(g);
|
||||
expect(g).toBeGreaterThan(b);
|
||||
expect(b).toBe(0);
|
||||
});
|
||||
|
||||
it("is monotonic: higher float16 input produces higher PQ output", () => {
|
||||
const dark = makeFloat16Frame(1, 1, { r: F16_ZERO, g: 0, b: 0, a: 0 });
|
||||
const mid = makeFloat16Frame(1, 1, { r: F16_HALF, g: 0, b: 0, a: 0 });
|
||||
const bright = makeFloat16Frame(1, 1, { r: F16_ONE, g: 0, b: 0, a: 0 });
|
||||
const r0 = float16ToPqRgb(dark, 8, 1, 1).readUInt16LE(0);
|
||||
const r1 = float16ToPqRgb(mid, 8, 1, 1).readUInt16LE(0);
|
||||
const r2 = float16ToPqRgb(bright, 8, 1, 1).readUInt16LE(0);
|
||||
expect(r0).toBe(0);
|
||||
expect(r1).toBeGreaterThan(r0);
|
||||
expect(r2).toBeGreaterThan(r1);
|
||||
});
|
||||
|
||||
it("is deterministic across calls with the same input", () => {
|
||||
const frame = makeFloat16Frame(3, 2, {
|
||||
r: F16_HALF,
|
||||
g: F16_ONE,
|
||||
b: F16_ZERO,
|
||||
a: F16_ONE,
|
||||
});
|
||||
const a = float16ToPqRgb(frame, 24, 3, 2);
|
||||
const b = float16ToPqRgb(frame, 24, 3, 2);
|
||||
expect(a.equals(b)).toBe(true);
|
||||
});
|
||||
|
||||
it("handles padded bytesPerRow (WebGPU 256-byte alignment)", () => {
|
||||
// WebGPU readback pads rows to 256-byte multiples. For a 4-pixel-wide
|
||||
// frame the actual pixel data is 32 bytes but bytesPerRow is 256.
|
||||
const width = 4;
|
||||
const height = 2;
|
||||
const bytesPerRow = 256;
|
||||
const frame = makeFloat16Frame(
|
||||
width,
|
||||
height,
|
||||
{ r: F16_HALF, g: F16_HALF, b: F16_HALF, a: 0 },
|
||||
bytesPerRow,
|
||||
);
|
||||
const out = float16ToPqRgb(frame, bytesPerRow, width, height);
|
||||
expect(out.length).toBe(width * height * 6);
|
||||
// Every R component should be the same non-zero value (uniform input).
|
||||
const expected = out.readUInt16LE(0);
|
||||
expect(expected).toBeGreaterThan(0);
|
||||
for (let pixel = 0; pixel < width * height; pixel++) {
|
||||
expect(out.readUInt16LE(pixel * 6)).toBe(expected);
|
||||
}
|
||||
});
|
||||
|
||||
it("ignores garbage bytes in the row padding region", () => {
|
||||
// Stuff junk into the trailing padding to make sure the PQ encoder
|
||||
// walks via bytesPerRow stride and not via raw buffer position.
|
||||
const width = 2;
|
||||
const height = 2;
|
||||
const bytesPerRow = 64;
|
||||
const frame = makeFloat16Frame(
|
||||
width,
|
||||
height,
|
||||
{ r: F16_ZERO, g: F16_ZERO, b: F16_ZERO, a: F16_ZERO },
|
||||
bytesPerRow,
|
||||
);
|
||||
for (let y = 0; y < height; y++) {
|
||||
const padStart = y * bytesPerRow + width * 8;
|
||||
for (let i = padStart; i < (y + 1) * bytesPerRow; i++) {
|
||||
frame[i] = 0xff;
|
||||
}
|
||||
}
|
||||
const out = float16ToPqRgb(frame, bytesPerRow, width, height);
|
||||
for (let i = 0; i < out.length; i += 2) {
|
||||
expect(out.readUInt16LE(i)).toBe(0);
|
||||
}
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user