fix(producer): cap the software encoder to the cores capture leaves free (#8019)

`canEncodeWhileCapturing` started an H.264 export beside the capture workers
whenever `workerCount < cpuCount`, but `appendSoftwareEncoderThreadsArg` gave the
encoder every usable core, so the overlap spawned a full-width x264 pool next to
the capture workers (#7458).

The overlap now reserves the cores capture is using: `softwareEncoderThreads =
max(1, cpuCount - workerCount)` travels through the streaming encoder options to
ffmpeg's `-threads`, while the post-capture encode keeps the old default.
This commit is contained in:
zhengqiuyang
2026-11-17 05:56:12 +00:00
committed by GitHub
parent 9daa7e876d
commit f89f6d3d4b
7 changed files with 70 additions and 5 deletions
@@ -37,6 +37,8 @@ export interface EncoderOptions {
* for compression and break concat-copy seams.
*/
lockGopForChunkConcat?: boolean;
softwareEncoderThreads?: number;
/**
* Required when `lockGopForChunkConcat` is `true`. Number of frames per
* GOP — set to `chunkSize` so every chunk starts on an IDR keyframe and
@@ -0,0 +1,21 @@
import { describe, expect, it } from "vitest";
import { buildStreamingArgs } from "./streamingEncoder.js";
const base = {
fps: { num: 30, den: 1 },
width: 64,
height: 64,
codec: "h264" as const,
preset: "medium",
quality: 16,
pixelFormat: "yuv420p",
};
describe("streaming encoder threads during a capture overlap", () => {
it("caps the software encoder to the cores the overlap reserves", () => {
const args = buildStreamingArgs({ ...base, softwareEncoderThreads: 2 }, "out.mp4");
const at = args.indexOf("-threads");
expect(at).toBeGreaterThan(-1);
expect(args[at + 1]).toBe("2");
});
});
@@ -186,6 +186,7 @@ export interface StreamingEncoderOptions {
lockGopForChunkConcat?: boolean;
/** Required when `lockGopForChunkConcat` is `true`. Frames per GOP. */
gopSize?: number;
softwareEncoderThreads?: number;
}
export interface StreamingEncoderResult {
@@ -364,7 +365,7 @@ export function buildStreamingArgs(
} else {
const encoderName = codec === "h264" ? "libx264" : "libx265";
args.push("-c:v", encoderName, "-preset", preset);
appendSoftwareEncoderThreadsArg(args, encoderName);
appendSoftwareEncoderThreadsArg(args, encoderName, options.softwareEncoderThreads);
if (bitrate) args.push("-b:v", bitrate);
else args.push("-crf", String(quality));
@@ -1044,7 +1044,10 @@ export async function renderChunk(
session = null;
verifyStarted = Date.now();
overlap.current = canEncodeWhileCapturing(encodeInput, chunkWorkerCount, usableCpuCount())
? encodeWhileCapturing(encodeInput, framesInChunk)
? encodeWhileCapturing(encodeInput, framesInChunk, {
workerCount: chunkWorkerCount,
cpuCount: usableCpuCount(),
})
: null;
await observeRenderStage(
new RenderObservabilityRecorder({ pipelineStartMs: start, log }),
@@ -268,7 +268,15 @@ function resolveEngineConfig(input: EncodeStageInput) {
return input.engineConfig ?? input.job.config.producerConfig ?? resolveConfig();
}
function videoEncoderOptions(input: EncodeStageInput, engineCfg: Pick<EngineConfig, "vp9CpuUsed">) {
export function overlapEncoderThreads(workerCount: number, cpuCount: number): number {
return Math.max(1, Math.floor(cpuCount) - Math.floor(workerCount));
}
export function videoEncoderOptions(
input: EncodeStageInput,
engineCfg: Pick<EngineConfig, "vp9CpuUsed">,
softwareEncoderThreads?: number,
) {
const { job, preset } = input;
return {
fps: job.config.fps,
@@ -289,6 +297,7 @@ function videoEncoderOptions(input: EncodeStageInput, engineCfg: Pick<EngineConf
// renders leave both undefined → preserves the existing open-GOP output.
lockGopForChunkConcat: input.lockGopForChunkConcat === true,
gopSize: input.gopSize,
softwareEncoderThreads,
};
}
@@ -343,8 +352,12 @@ async function readLandedFrame(workerPath: string | undefined, mergedPath: strin
export function encodeWhileCapturing(
input: EncodeStageInput,
totalFrames: number,
captureBudget?: { workerCount: number; cpuCount: number },
): EncodeWhileCapturing {
const engineCfg = resolveEngineConfig(input);
const softwareEncoderThreads = captureBudget
? overlapEncoderThreads(captureBudget.workerCount, captureBudget.cpuCount)
: undefined;
const stop = new AbortController();
const signal = input.abortSignal
? AbortSignal.any([input.abortSignal, stop.signal])
@@ -362,7 +375,10 @@ export function encodeWhileCapturing(
const encoder = spawnStreamingEncoder(
input.videoOnlyPath,
{ ...videoEncoderOptions(input, engineCfg), imageFormat: "jpeg" },
{
...videoEncoderOptions(input, engineCfg, softwareEncoderThreads),
imageFormat: "jpeg",
},
signal,
engineCfg,
);
@@ -8,7 +8,9 @@ import { getFfmpegBinary } from "@hyperframes/engine";
import {
canEncodeWhileCapturing,
encodeWhileCapturing,
overlapEncoderThreads,
runEncodeStage,
videoEncoderOptions,
type EncodeStageInput,
} from "./encodeStage.js";
@@ -104,6 +106,23 @@ describe("encodeWhileCapturing", () => {
).toBe(true);
});
it("caps the software encoder to the cores capture leaves free", () => {
expect(overlapEncoderThreads(4, 8)).toBe(4);
expect(overlapEncoderThreads(6, 8)).toBe(2);
expect(overlapEncoderThreads(7, 8)).toBe(1);
// Never zero, even when the caller passes every core to capture.
expect(overlapEncoderThreads(8, 8)).toBe(1);
const input = makeInput("/r", "/r/frames", "v.mp4");
expect(
videoEncoderOptions(input, { vp9CpuUsed: undefined }, overlapEncoderThreads(6, 8)),
).toMatchObject({ softwareEncoderThreads: 2 });
// The post-capture encode keeps the encoder default (all usable cores).
expect(videoEncoderOptions(input, { vp9CpuUsed: undefined })).toMatchObject({
softwareEncoderThreads: undefined,
});
});
it("encodes a closed-GOP distributed chunk byte-identical to the encode after capture", async () => {
const { root, framesDir, landed, merge } = captureToWorkerDirs();
const locked = (name: string) => ({
@@ -5580,7 +5580,10 @@ async function executeRenderPipeline(input: {
!cfg.useDrawElement &&
workerSizing !== undefined &&
canEncodeWhileCapturing(encodeInput, capturePlan.workerCount, workerSizing.cpuCount)
? encodeWhileCapturing(encodeInput, totalFrames)
? encodeWhileCapturing(encodeInput, totalFrames, {
workerCount: capturePlan.workerCount,
cpuCount: workerSizing.cpuCount,
})
: null;
const captureFrameStart = Date.now();
const invokeDiskCapture = (diskPlan: SdrDiskCapturePlan) =>