mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-04 13:07:11 +00:00
fix(engine): treat ffmpegStreamingTimeout as per-frame inactivity, not total render time (#901)
## Summary - Convert `streamingEncoder.ts`'s safety timer from a total-render hard cap to a per-frame inactivity timeout - Reset the timer only on `accepted === true` writes — buffered writes don't count as consumer progress - Update the `ffmpegStreamingTimeout` config doc to reflect the new semantics ## The bug The timer was set once at spawn and fired SIGTERM unconditionally at `ffmpegStreamingTimeout` ms — turning a "FFmpeg is hung" guard into a hard cap on total render duration. Slow-but-progressing captures (CI runner under load, large compositions, slower compositor paths after [#838](https://github.com/heygen-com/hyperframes/pull/838)'s always-clip change) regularly exceeded the 600s default and were killed mid-encode. The symptom surfaced as: ``` Streaming encode failed: FFmpeg exited with code 255 video:NNNkB audio:0kB ... [libx264 @ ...] frame I:3 Avg QP:12.91 size: 73263 [libx264 @ ...] frame P:431 Avg QP:14.72 size: 31633 ... [libx264 @ ...] kb/s:7661.05 Exiting normally, received signal 15. ``` libx264 had encoded most frames cleanly; SIGTERM arrived during the encode, libx264 printed its end-of-encode stats, and Node observed a non-zero exit. The `audio:0kB` in stderr is incidental — `streamingEncoder` is video-only; audio is muxed later in `assembleStage`. Downstream reproduction: `style-13-prod` fails deterministically in `heygen-com/hyperframes-internal` CI after bumping `@hyperframes/producer` from 0.6.7 → 0.6.10. Bisects to #838 widening the SDR capture path at dpr=1 — same composition shape, slower per-frame, total render now crosses 600s. ## The fix Convert the timer to a heartbeat: each `writeFrame` that goes through to the kernel pipe (i.e. `stdin.write` returns `true`) resets it. Only true hangs (no successful frame write for the timeout window) trip SIGTERM now; "slow but progressing" renders are unbounded. Crucially, the heartbeat does **not** reset on `accepted === false`. A `false` return means Node had to buffer the write because FFmpeg hasn't drained the pipe yet — that's not proof of consumer progress, just proof we produced. Without this distinction, a hung FFmpeg with a live Chrome would queue frames into Node's writable buffer indefinitely (no backpressure path back to the capture loop) and grow until OOM. In steady state with a slow-but-alive FFmpeg, writes alternate between `true` and `false` as the buffer drains and refills; the `true`s are enough to keep the heartbeat ticking. Renames are intentionally avoided — `ffmpegStreamingTimeout` keeps its name and `600_000` default; only the semantics changed. The config doc spells out the new behavior so downstream consumers know what 600s now means. ## Test plan - [x] **Slow-but-progressing capture** (`accepted=true`): 9× `writeFrame` at 900ms intervals (under the 1000ms threshold) — encoder stays alive through 8.1s. Stall past the threshold — SIGTERM fires. - [x] **Stalled FFmpeg with live producer** (`accepted=false`): override `stdin.write` to return false; pump 9× `writeFrame` at 900ms intervals. SIGTERM still fires inside the 1000ms window — buffered writes don't keep the heartbeat alive. - [x] Existing 33 tests in `streamingEncoder.test.ts` still pass - [x] Lint (`oxlint`) + format (`oxfmt --check`) clean - [ ] CI regression suite 🤖 Generated with [Claude Code](https://claude.com/claude-code)
This commit is contained in:
@@ -94,7 +94,12 @@ export interface EngineConfig {
|
|||||||
ffmpegEncodeTimeout: number;
|
ffmpegEncodeTimeout: number;
|
||||||
/** Timeout for FFmpeg mux/faststart processes (ms). Default: 300_000 */
|
/** Timeout for FFmpeg mux/faststart processes (ms). Default: 300_000 */
|
||||||
ffmpegProcessTimeout: number;
|
ffmpegProcessTimeout: number;
|
||||||
/** Timeout for FFmpeg streaming encode (ms). Default: 600_000 */
|
/**
|
||||||
|
* Inactivity timeout for FFmpeg streaming encode (ms). The timer resets on
|
||||||
|
* every successful `writeFrame` call, so this caps the duration of a
|
||||||
|
* single "no frame arrived" gap (capture hang, dead Chrome), not the total
|
||||||
|
* render time. Default: 600_000 (10 minutes without any frame = dead).
|
||||||
|
*/
|
||||||
ffmpegStreamingTimeout: number;
|
ffmpegStreamingTimeout: number;
|
||||||
|
|
||||||
// ── HDR ──────────────────────────────────────────────────────────────
|
// ── HDR ──────────────────────────────────────────────────────────────
|
||||||
|
|||||||
@@ -572,4 +572,71 @@ describe("spawnStreamingEncoder lifecycle and cleanup", () => {
|
|||||||
controller.abort();
|
controller.abort();
|
||||||
expect(proc.kill).not.toHaveBeenCalled();
|
expect(proc.kill).not.toHaveBeenCalled();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("inactivity timeout fires only after a no-frame gap exceeds ffmpegStreamingTimeout", async () => {
|
||||||
|
vi.useFakeTimers();
|
||||||
|
try {
|
||||||
|
const { spawn, calls } = createSpawnSpy();
|
||||||
|
vi.resetModules();
|
||||||
|
vi.doMock("child_process", () => ({ spawn }));
|
||||||
|
|
||||||
|
const { spawnStreamingEncoder } = await import("./streamingEncoder.js");
|
||||||
|
const dir = mkdtempSync(join(tmpdir(), "se-heartbeat-"));
|
||||||
|
const encoder = await spawnStreamingEncoder(join(dir, "out.mp4"), baseOptions, undefined, {
|
||||||
|
ffmpegStreamingTimeout: 1000,
|
||||||
|
});
|
||||||
|
|
||||||
|
const proc = calls[0]!.proc;
|
||||||
|
|
||||||
|
// Frames every 900ms — under the 1000ms inactivity threshold — should
|
||||||
|
// keep resetting the timer. After 9× 900ms = 8.1s of "slow but
|
||||||
|
// progressing" capture the encoder must still be alive. The old total-
|
||||||
|
// render timeout would have fired SIGTERM at ~1000ms.
|
||||||
|
for (let i = 0; i < 9; i++) {
|
||||||
|
encoder.writeFrame(Buffer.from([i]));
|
||||||
|
vi.advanceTimersByTime(900);
|
||||||
|
}
|
||||||
|
expect(proc.kill).not.toHaveBeenCalled();
|
||||||
|
|
||||||
|
// Now stall — no writeFrame for longer than the threshold. SIGTERM fires.
|
||||||
|
vi.advanceTimersByTime(1100);
|
||||||
|
expect(proc.kill).toHaveBeenCalledWith("SIGTERM");
|
||||||
|
} finally {
|
||||||
|
vi.useRealTimers();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it("inactivity timeout still fires when stdin is backpressured (stalled ffmpeg, live producer)", async () => {
|
||||||
|
vi.useFakeTimers();
|
||||||
|
try {
|
||||||
|
// Simulate the FFmpeg-hangs-but-Chrome-keeps-producing case: stdin.write
|
||||||
|
// always returns false (Node has to buffer because ffmpeg isn't draining
|
||||||
|
// the pipe). The heartbeat must NOT reset on those buffered writes —
|
||||||
|
// otherwise a hung ffmpeg with a steady frame producer would never
|
||||||
|
// SIGTERM and we'd grow Node's stdin buffer until OOM.
|
||||||
|
const { spawn, calls } = createSpawnSpy();
|
||||||
|
vi.resetModules();
|
||||||
|
vi.doMock("child_process", () => ({ spawn }));
|
||||||
|
|
||||||
|
const { spawnStreamingEncoder } = await import("./streamingEncoder.js");
|
||||||
|
const dir = mkdtempSync(join(tmpdir(), "se-backpressure-"));
|
||||||
|
const encoder = await spawnStreamingEncoder(join(dir, "out.mp4"), baseOptions, undefined, {
|
||||||
|
ffmpegStreamingTimeout: 1000,
|
||||||
|
});
|
||||||
|
|
||||||
|
const proc = calls[0]!.proc;
|
||||||
|
proc.stdin.write = (_chunk: Buffer) => false;
|
||||||
|
|
||||||
|
// Pump 9 frames at 900ms intervals — all returning false. The reset
|
||||||
|
// should NOT fire (every write was buffered, not accepted), so the
|
||||||
|
// 1000ms timer (last reset on spawn) elapses near the start.
|
||||||
|
for (let i = 0; i < 9; i++) {
|
||||||
|
encoder.writeFrame(Buffer.from([i]));
|
||||||
|
vi.advanceTimersByTime(900);
|
||||||
|
}
|
||||||
|
expect(proc.kill).toHaveBeenCalledWith("SIGTERM");
|
||||||
|
} finally {
|
||||||
|
vi.useRealTimers();
|
||||||
|
}
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -416,13 +416,26 @@ export async function spawnStreamingEncoder(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Timeout safety
|
// Inactivity timeout: fires only when no frame has been written for
|
||||||
|
// `ffmpegStreamingTimeout` ms. A slow-but-progressing capture (e.g. a CI
|
||||||
|
// runner under load) keeps resetting the timer on each writeFrame, so total
|
||||||
|
// wall-clock render time is unbounded — only a true hang (Chrome dead,
|
||||||
|
// capture stuck, no frames arriving) trips SIGTERM. The 600s default was
|
||||||
|
// previously a total-render cap, which intermittently killed legitimate
|
||||||
|
// slow renders mid-encode (FFmpeg got SIGTERM after most frames were sent;
|
||||||
|
// libx264 printed its summary and exited 255, observable as
|
||||||
|
// "Streaming encode failed: FFmpeg exited with code 255" with audio:0kB).
|
||||||
const streamingTimeout = config?.ffmpegStreamingTimeout ?? DEFAULT_CONFIG.ffmpegStreamingTimeout;
|
const streamingTimeout = config?.ffmpegStreamingTimeout ?? DEFAULT_CONFIG.ffmpegStreamingTimeout;
|
||||||
const timer = setTimeout(() => {
|
let timer: NodeJS.Timeout | null = null;
|
||||||
if (exitStatus === "running") {
|
const resetTimer = () => {
|
||||||
ffmpeg.kill("SIGTERM");
|
if (timer) clearTimeout(timer);
|
||||||
}
|
timer = setTimeout(() => {
|
||||||
}, streamingTimeout);
|
if (exitStatus === "running") {
|
||||||
|
ffmpeg.kill("SIGTERM");
|
||||||
|
}
|
||||||
|
}, streamingTimeout);
|
||||||
|
};
|
||||||
|
resetTimer();
|
||||||
|
|
||||||
const encoder: StreamingEncoder = {
|
const encoder: StreamingEncoder = {
|
||||||
writeFrame: (buffer: Buffer): boolean => {
|
writeFrame: (buffer: Buffer): boolean => {
|
||||||
@@ -433,11 +446,20 @@ export async function spawnStreamingEncoder(
|
|||||||
// provided buffer and drain it asynchronously. The HDR path's compositor
|
// provided buffer and drain it asynchronously. The HDR path's compositor
|
||||||
// reuses pre-allocated transOutput/normalCanvas buffers across frames,
|
// reuses pre-allocated transOutput/normalCanvas buffers across frames,
|
||||||
// so without this copy the pipe would read partially-overwritten data
|
// so without this copy the pipe would read partially-overwritten data
|
||||||
// and flicker. The SDR path doesn't invoke writeFrame at all (it pipes
|
// and flicker.
|
||||||
// PNG files via encodeFramesFromDir), so the memcpy here is HDR-only
|
|
||||||
// and justified by correctness.
|
|
||||||
const copy = Buffer.from(buffer);
|
const copy = Buffer.from(buffer);
|
||||||
return ffmpeg.stdin.write(copy);
|
const accepted = ffmpeg.stdin.write(copy);
|
||||||
|
// Reset inactivity timer ONLY on `accepted === true`. `true` means the
|
||||||
|
// write went through to the kernel pipe without buffering in Node —
|
||||||
|
// proof FFmpeg is actually consuming. `false` means Node's writable
|
||||||
|
// stream had to buffer (FFmpeg hasn't drained the pipe yet); we deliberately
|
||||||
|
// don't reset on `false` so a hung FFmpeg with a still-producing Chrome
|
||||||
|
// can't keep us alive forever while Node's stdin buffer grows to OOM. In
|
||||||
|
// steady state with a slower-but-alive FFmpeg, writes alternate between
|
||||||
|
// true and false as the buffer drains and refills; the trues are enough
|
||||||
|
// to keep the heartbeat ticking.
|
||||||
|
if (accepted) resetTimer();
|
||||||
|
return accepted;
|
||||||
},
|
},
|
||||||
|
|
||||||
close: async (): Promise<StreamingEncoderResult> => {
|
close: async (): Promise<StreamingEncoderResult> => {
|
||||||
@@ -455,7 +477,10 @@ export async function spawnStreamingEncoder(
|
|||||||
// repeated calls. If you change this method, preserve idempotency or
|
// repeated calls. If you change this method, preserve idempotency or
|
||||||
// a regression here will silently double-close ffmpeg and produce
|
// a regression here will silently double-close ffmpeg and produce
|
||||||
// harder-to-trace errors at the orchestrator layer.
|
// harder-to-trace errors at the orchestrator layer.
|
||||||
clearTimeout(timer);
|
if (timer) {
|
||||||
|
clearTimeout(timer);
|
||||||
|
timer = null;
|
||||||
|
}
|
||||||
if (signal) signal.removeEventListener("abort", onAbort);
|
if (signal) signal.removeEventListener("abort", onAbort);
|
||||||
|
|
||||||
const stdin = ffmpeg.stdin;
|
const stdin = ffmpeg.stdin;
|
||||||
|
|||||||
Reference in New Issue
Block a user