perf(producer): size a long video's chunks in one plan, no re-plan (#7382)

* perf(producer): size a long video's chunks in one plan, no re-plan

* docs(producer): chunk formulas name the minParallelChunks floor
This commit is contained in:
Miguel Ángel
2026-11-07 21:29:51 +00:00
committed by GitHub
parent 3d7f12acd7
commit c27475b972
6 changed files with 75 additions and 5 deletions
@@ -43,6 +43,8 @@ function request() {
codec: "h264",
chunkSize: 120,
maxParallelChunks: 8,
targetChunkFrames: 1600,
minParallelChunks: 4,
cfr: true,
},
},
+4
View File
@@ -32,6 +32,7 @@ export interface DistributedRenderOptions {
chunkSize?: number;
maxParallelChunks?: number;
targetChunkFrames?: number;
minParallelChunks?: number;
runtimeCap?: DistributedRenderConfig["runtimeCap"];
rejectOnSystemFonts?: boolean;
failClosedFontFetch?: boolean;
@@ -166,6 +167,7 @@ function assertDistributedOptions(value: unknown): void {
"chunkSize",
"maxParallelChunks",
"targetChunkFrames",
"minParallelChunks",
"planDirSizeLimitBytes",
] as const) {
assertOptionalInteger(value, field, 1);
@@ -328,6 +330,7 @@ export function distributedConfigFromRequest(
chunkSize: distributed.chunkSize,
maxParallelChunks: distributed.maxParallelChunks,
targetChunkFrames: distributed.targetChunkFrames,
minParallelChunks: distributed.minParallelChunks,
runtimeCap: distributed.runtimeCap,
rejectOnSystemFonts: distributed.rejectOnSystemFonts,
failClosedFontFetch: distributed.failClosedFontFetch,
@@ -364,6 +367,7 @@ export function renderRequestFromDistributedConfig(input: {
...optionalProperty("chunkSize", config.chunkSize),
...optionalProperty("maxParallelChunks", config.maxParallelChunks),
...optionalProperty("targetChunkFrames", config.targetChunkFrames),
...optionalProperty("minParallelChunks", config.minParallelChunks),
...optionalProperty("runtimeCap", config.runtimeCap),
...optionalProperty("rejectOnSystemFonts", config.rejectOnSystemFonts),
...optionalProperty("failClosedFontFetch", config.failClosedFontFetch),
@@ -391,6 +391,34 @@ describe("resolveChunkPlan", () => {
expect(result.effectiveChunkSize).toBe(240);
});
describe("sized in one plan from a frame budget and a fan-out floor", () => {
// The workflow's budget is 1600 frames a chunk and its fan-out 4 chunks, up to a tier cap of 16.
const size = (totalFrames: number, cap = 16) =>
resolveChunkPlan(totalFrames, undefined, cap, 1600, 4).chunkCount;
it("adds chunks to a long film so each stays inside the budget", () => {
expect(size(10929)).toBe(7);
expect(
resolveChunkPlan(10929, undefined, 16, 1600, 4).effectiveChunkSize,
).toBeLessThanOrEqual(1600);
});
it("keeps a short film at the fan-out instead of collapsing it to the budget", () => {
expect(size(1800)).toBe(4);
expect(size(6400)).toBe(4);
});
it("rejects a non-positive or non-integer floor", () => {
expect(() => resolveChunkPlan(1800, undefined, 16, 1600, 0)).toThrow(/minParallelChunks/);
expect(() => resolveChunkPlan(1800, undefined, 16, 1600, 2.5)).toThrow(/minParallelChunks/);
});
it("never goes past the cap, however long the film", () => {
expect(size(54000, 9)).toBe(9);
expect(size(1800, 2)).toBe(2);
});
});
it("rejects non-positive totalFrames", () => {
expect(() => resolveChunkPlan(0, 240, 16)).toThrow();
expect(() => resolveChunkPlan(-1, 240, 16)).toThrow();
@@ -221,17 +221,19 @@ export interface DistributedRenderConfig extends Pick<RenderRequestOptions, "pag
* chunks whose per-chunk frame count stays at or below this bound, still
* capped by `maxParallelChunks`:
*
* chunkCount = clamp(ceil(totalFrames / targetChunkFrames), 1, maxParallelChunks)
* chunkCount = clamp(ceil(totalFrames / targetChunkFrames), minParallelChunks, maxParallelChunks)
*
* This bounds per-chunk render *time* (which scales with frames-per-chunk) so
* a single chunk can't exceed a downstream per-chunk timeout on a long video,
* while short videos still collapse to few chunks. It is a ceiling, not a
* while short videos drop to `minParallelChunks` (default 1). It is a ceiling, not a
* fixed size: a video short enough to fit in fewer chunks gets fewer. Ignored
* when `chunkSize` is set (an explicit fixed size already pins per-chunk
* frames). Mutually exclusive with `chunkSize` in intent; if both are passed,
* `chunkSize` wins and `targetChunkFrames` is a no-op.
*/
targetChunkFrames?: number;
/** Fewest chunks a `targetChunkFrames`-sized plan uses, so a short video keeps this fan-out. */
minParallelChunks?: number;
/** Runtime hint; consumed by future per-runtime budget checks. The current implementation records the value but does not enforce. */
runtimeCap?: "lambda" | "temporal" | "cloud-run-job" | "k8s-job" | "none";
@@ -767,8 +769,8 @@ function assertPlanSizeWithinLimit(input: {
* on tiny renders). Explicit numbers, including `240`, take precedence.
*
* Optional `targetChunkFrames` caps per-chunk frames in the auto-sized path:
* the auto-sizer then targets `clamp(ceil(totalFrames / targetChunkFrames), 1,
* maxParallelChunks)` chunks, so short videos collapse to fewer chunks and long
* the auto-sizer then targets `clamp(ceil(totalFrames / targetChunkFrames), minParallelChunks,
* maxParallelChunks)` chunks, so short videos drop to `minParallelChunks` and long
* videos add chunks (up to the cap) to keep each one under the bound. It is a
* no-op when omitted, and ignored when `configChunkSize` is set.
*/
@@ -777,6 +779,7 @@ export function resolveChunkPlan(
configChunkSize: number | undefined,
maxParallelChunks: number,
targetChunkFrames?: number,
minParallelChunks = 1,
): { chunkCount: number; effectiveChunkSize: number } {
// Integer-only inputs: a fractional `totalFrames` (e.g. 10.5) would
// otherwise produce a last chunk with non-integer `endFrame`, and the
@@ -795,6 +798,7 @@ export function resolveChunkPlan(
if (targetChunkFrames !== undefined) {
assertPositiveInteger("targetChunkFrames", targetChunkFrames);
}
assertPositiveInteger("minParallelChunks", minParallelChunks);
// `targetChunkFrames` lowers the auto-sizer's effective parallelism so the
// chosen chunk count keeps frames-per-chunk at or below the bound, without
// ever exceeding `maxParallelChunks`. It only affects the auto-sized path
@@ -805,7 +809,10 @@ export function resolveChunkPlan(
const autoSizeParallel =
targetChunkFrames === undefined
? maxParallelChunks
: Math.min(maxParallelChunks, Math.max(1, Math.ceil(totalFrames / targetChunkFrames)));
: Math.min(
maxParallelChunks,
Math.max(minParallelChunks, Math.ceil(totalFrames / targetChunkFrames)),
);
const resolvedChunkSize =
configChunkSize ?? Math.max(MIN_CHUNK_SIZE, Math.ceil(totalFrames / autoSizeParallel));
const naiveCount = Math.ceil(totalFrames / resolvedChunkSize);
@@ -1673,6 +1680,7 @@ export async function buildLocalExecutionPlan(
config.chunkSize,
maxParallel,
config.targetChunkFrames,
config.minParallelChunks,
);
const chunks = buildPlanChunkSlices(
frameWeights,
@@ -19,3 +19,16 @@ describe("validateDistributedRenderConfig proresProfile", () => {
).toThrow("applies to mov renders, not mp4");
});
});
describe("validateDistributedRenderConfig minParallelChunks", () => {
const mp4 = { ...base, format: "mp4" } as const;
it("accepts a floor up to the chunk ceiling and refuses one outside it", () => {
expect(() => validateDistributedRenderConfig({ ...mp4, minParallelChunks: 4 })).not.toThrow();
expect(() => validateDistributedRenderConfig({ ...mp4, minParallelChunks: 0 })).toThrow(
"config.minParallelChunks",
);
expect(() => validateDistributedRenderConfig({ ...mp4, minParallelChunks: 257 })).toThrow(
"must be <= 256",
);
});
});
@@ -222,6 +222,21 @@ export function validateDistributedRenderConfig(
}
}
if (config.minParallelChunks !== undefined) {
if (!Number.isInteger(config.minParallelChunks) || config.minParallelChunks < 1) {
throw new InvalidConfigError(
"config.minParallelChunks",
`must be a positive integer; got ${config.minParallelChunks}`,
);
}
if (config.minParallelChunks > MAX_PARALLEL_CHUNKS_CEILING) {
throw new InvalidConfigError(
"config.minParallelChunks",
`must be <= ${MAX_PARALLEL_CHUNKS_CEILING}; got ${config.minParallelChunks}`,
);
}
}
for (const field of ["workerTimeoutSeconds", "workerConcurrency"] as const) {
const value = config[field];
if (value !== undefined && (!Number.isInteger(value) || value < 1)) {