mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-18 07:50:13 +00:00
fix(producer): cap the software encoder to the cores capture leaves free (#8019)
`canEncodeWhileCapturing` started an H.264 export beside the capture workers whenever `workerCount < cpuCount`, but `appendSoftwareEncoderThreadsArg` gave the encoder every usable core, so the overlap spawned a full-width x264 pool next to the capture workers (#7458). The overlap now reserves the cores capture is using: `softwareEncoderThreads = max(1, cpuCount - workerCount)` travels through the streaming encoder options to ffmpeg's `-threads`, while the post-capture encode keeps the old default.
This commit is contained in:
@@ -37,6 +37,8 @@ export interface EncoderOptions {
|
||||
* for compression and break concat-copy seams.
|
||||
*/
|
||||
lockGopForChunkConcat?: boolean;
|
||||
softwareEncoderThreads?: number;
|
||||
|
||||
/**
|
||||
* Required when `lockGopForChunkConcat` is `true`. Number of frames per
|
||||
* GOP — set to `chunkSize` so every chunk starts on an IDR keyframe and
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { buildStreamingArgs } from "./streamingEncoder.js";
|
||||
|
||||
const base = {
|
||||
fps: { num: 30, den: 1 },
|
||||
width: 64,
|
||||
height: 64,
|
||||
codec: "h264" as const,
|
||||
preset: "medium",
|
||||
quality: 16,
|
||||
pixelFormat: "yuv420p",
|
||||
};
|
||||
|
||||
describe("streaming encoder threads during a capture overlap", () => {
|
||||
it("caps the software encoder to the cores the overlap reserves", () => {
|
||||
const args = buildStreamingArgs({ ...base, softwareEncoderThreads: 2 }, "out.mp4");
|
||||
const at = args.indexOf("-threads");
|
||||
expect(at).toBeGreaterThan(-1);
|
||||
expect(args[at + 1]).toBe("2");
|
||||
});
|
||||
});
|
||||
@@ -186,6 +186,7 @@ export interface StreamingEncoderOptions {
|
||||
lockGopForChunkConcat?: boolean;
|
||||
/** Required when `lockGopForChunkConcat` is `true`. Frames per GOP. */
|
||||
gopSize?: number;
|
||||
softwareEncoderThreads?: number;
|
||||
}
|
||||
|
||||
export interface StreamingEncoderResult {
|
||||
@@ -364,7 +365,7 @@ export function buildStreamingArgs(
|
||||
} else {
|
||||
const encoderName = codec === "h264" ? "libx264" : "libx265";
|
||||
args.push("-c:v", encoderName, "-preset", preset);
|
||||
appendSoftwareEncoderThreadsArg(args, encoderName);
|
||||
appendSoftwareEncoderThreadsArg(args, encoderName, options.softwareEncoderThreads);
|
||||
if (bitrate) args.push("-b:v", bitrate);
|
||||
else args.push("-crf", String(quality));
|
||||
|
||||
|
||||
@@ -1044,7 +1044,10 @@ export async function renderChunk(
|
||||
session = null;
|
||||
verifyStarted = Date.now();
|
||||
overlap.current = canEncodeWhileCapturing(encodeInput, chunkWorkerCount, usableCpuCount())
|
||||
? encodeWhileCapturing(encodeInput, framesInChunk)
|
||||
? encodeWhileCapturing(encodeInput, framesInChunk, {
|
||||
workerCount: chunkWorkerCount,
|
||||
cpuCount: usableCpuCount(),
|
||||
})
|
||||
: null;
|
||||
await observeRenderStage(
|
||||
new RenderObservabilityRecorder({ pipelineStartMs: start, log }),
|
||||
|
||||
@@ -268,7 +268,15 @@ function resolveEngineConfig(input: EncodeStageInput) {
|
||||
return input.engineConfig ?? input.job.config.producerConfig ?? resolveConfig();
|
||||
}
|
||||
|
||||
function videoEncoderOptions(input: EncodeStageInput, engineCfg: Pick<EngineConfig, "vp9CpuUsed">) {
|
||||
export function overlapEncoderThreads(workerCount: number, cpuCount: number): number {
|
||||
return Math.max(1, Math.floor(cpuCount) - Math.floor(workerCount));
|
||||
}
|
||||
|
||||
export function videoEncoderOptions(
|
||||
input: EncodeStageInput,
|
||||
engineCfg: Pick<EngineConfig, "vp9CpuUsed">,
|
||||
softwareEncoderThreads?: number,
|
||||
) {
|
||||
const { job, preset } = input;
|
||||
return {
|
||||
fps: job.config.fps,
|
||||
@@ -289,6 +297,7 @@ function videoEncoderOptions(input: EncodeStageInput, engineCfg: Pick<EngineConf
|
||||
// renders leave both undefined → preserves the existing open-GOP output.
|
||||
lockGopForChunkConcat: input.lockGopForChunkConcat === true,
|
||||
gopSize: input.gopSize,
|
||||
softwareEncoderThreads,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -343,8 +352,12 @@ async function readLandedFrame(workerPath: string | undefined, mergedPath: strin
|
||||
export function encodeWhileCapturing(
|
||||
input: EncodeStageInput,
|
||||
totalFrames: number,
|
||||
captureBudget?: { workerCount: number; cpuCount: number },
|
||||
): EncodeWhileCapturing {
|
||||
const engineCfg = resolveEngineConfig(input);
|
||||
const softwareEncoderThreads = captureBudget
|
||||
? overlapEncoderThreads(captureBudget.workerCount, captureBudget.cpuCount)
|
||||
: undefined;
|
||||
const stop = new AbortController();
|
||||
const signal = input.abortSignal
|
||||
? AbortSignal.any([input.abortSignal, stop.signal])
|
||||
@@ -362,7 +375,10 @@ export function encodeWhileCapturing(
|
||||
|
||||
const encoder = spawnStreamingEncoder(
|
||||
input.videoOnlyPath,
|
||||
{ ...videoEncoderOptions(input, engineCfg), imageFormat: "jpeg" },
|
||||
{
|
||||
...videoEncoderOptions(input, engineCfg, softwareEncoderThreads),
|
||||
imageFormat: "jpeg",
|
||||
},
|
||||
signal,
|
||||
engineCfg,
|
||||
);
|
||||
|
||||
@@ -8,7 +8,9 @@ import { getFfmpegBinary } from "@hyperframes/engine";
|
||||
import {
|
||||
canEncodeWhileCapturing,
|
||||
encodeWhileCapturing,
|
||||
overlapEncoderThreads,
|
||||
runEncodeStage,
|
||||
videoEncoderOptions,
|
||||
type EncodeStageInput,
|
||||
} from "./encodeStage.js";
|
||||
|
||||
@@ -104,6 +106,23 @@ describe("encodeWhileCapturing", () => {
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it("caps the software encoder to the cores capture leaves free", () => {
|
||||
expect(overlapEncoderThreads(4, 8)).toBe(4);
|
||||
expect(overlapEncoderThreads(6, 8)).toBe(2);
|
||||
expect(overlapEncoderThreads(7, 8)).toBe(1);
|
||||
// Never zero, even when the caller passes every core to capture.
|
||||
expect(overlapEncoderThreads(8, 8)).toBe(1);
|
||||
|
||||
const input = makeInput("/r", "/r/frames", "v.mp4");
|
||||
expect(
|
||||
videoEncoderOptions(input, { vp9CpuUsed: undefined }, overlapEncoderThreads(6, 8)),
|
||||
).toMatchObject({ softwareEncoderThreads: 2 });
|
||||
// The post-capture encode keeps the encoder default (all usable cores).
|
||||
expect(videoEncoderOptions(input, { vp9CpuUsed: undefined })).toMatchObject({
|
||||
softwareEncoderThreads: undefined,
|
||||
});
|
||||
});
|
||||
|
||||
it("encodes a closed-GOP distributed chunk byte-identical to the encode after capture", async () => {
|
||||
const { root, framesDir, landed, merge } = captureToWorkerDirs();
|
||||
const locked = (name: string) => ({
|
||||
|
||||
@@ -5580,7 +5580,10 @@ async function executeRenderPipeline(input: {
|
||||
!cfg.useDrawElement &&
|
||||
workerSizing !== undefined &&
|
||||
canEncodeWhileCapturing(encodeInput, capturePlan.workerCount, workerSizing.cpuCount)
|
||||
? encodeWhileCapturing(encodeInput, totalFrames)
|
||||
? encodeWhileCapturing(encodeInput, totalFrames, {
|
||||
workerCount: capturePlan.workerCount,
|
||||
cpuCount: workerSizing.cpuCount,
|
||||
})
|
||||
: null;
|
||||
const captureFrameStart = Date.now();
|
||||
const invokeDiskCapture = (diskPlan: SdrDiskCapturePlan) =>
|
||||
|
||||
Reference in New Issue
Block a user