perf(producer): size distributed chunks to the worker limit and concurrency (#5765)

* perf(producer): size distributed chunks to the worker limit and concurrency

* chore(cli): drop a comment the field name already says
This commit is contained in:
Miguel Ángel
2026-10-17 08:45:47 +00:00
committed by GitHub
parent d5162beb50
commit c69fd2c6ec
18 changed files with 378 additions and 30 deletions
@@ -86,6 +86,7 @@ export async function runDeploy(args: Partial<DeployArgs> = {}): Promise<void> {
stateMachineArn: outputs.stateMachineArn,
functionName: outputs.functionName,
lambdaMemoryMb: resolved.lambdaMemoryMb,
reservedConcurrency: resolved.reservedConcurrency,
deployedAt: new Date().toISOString(),
});
@@ -167,4 +167,8 @@ describe("buildLambdaBatchRenderConfig — aspect-agnostic wire threading", () =
expect(config.outputResolution).toBe("portrait-4k");
expect(config.outputResolutionAspectAgnostic).toBeUndefined();
});
it("hands the deploy's reserved concurrency to the planner", () => {
expect(buildLambdaBatchRenderConfig(baseArgs, 64).workerConcurrency).toBe(64);
});
});
@@ -217,7 +217,10 @@ export async function runRenderBatch(args: RenderBatchArgs): Promise<void> {
failCommand();
}
const config: SerializableDistributedRenderConfig = buildLambdaBatchRenderConfig(args);
const config: SerializableDistributedRenderConfig = buildLambdaBatchRenderConfig(
args,
stack.reservedConcurrency,
);
// Deploy the site once and reuse it across every entry. --site-id and
// --dry-run both skip the deploy via a synthesised handle.
@@ -354,6 +357,7 @@ function makePlaceholderSiteHandle(siteId: string, bucketName: string): SiteHand
*/
export function buildLambdaBatchRenderConfig(
args: RenderBatchArgs,
workerConcurrency?: number,
): SerializableDistributedRenderConfig {
const config: SerializableDistributedRenderConfig = {
fps: args.fps,
@@ -366,6 +370,7 @@ export function buildLambdaBatchRenderConfig(
chunkSize: args.chunkSize,
maxParallelChunks: args.maxParallelChunks,
targetChunkFrames: args.targetChunkFrames,
workerConcurrency,
runtimeCap: "lambda",
};
if (args.outputResolutionAspectAgnostic) {
@@ -10,6 +10,7 @@
*/
import { describe, expect, it } from "vitest";
import lambdaCommand from "../lambda.js";
import { buildLambdaRenderConfig, type RenderArgs } from "./render.js";
const BASE_ARGS: RenderArgs = {
@@ -55,3 +56,18 @@ describe("buildLambdaRenderConfig — aspect-agnostic wire threading", () => {
expect(config.variables).toEqual({ alice: "hello" });
});
});
describe("lambda render chunk sizing", () => {
it("hands the deploy's reserved concurrency to the planner", () => {
expect(buildLambdaRenderConfig(BASE_ARGS, undefined, 64).workerConcurrency).toBe(64);
expect(buildLambdaRenderConfig(BASE_ARGS, undefined).workerConcurrency).toBeUndefined();
});
it("help no longer claims --chunk-size defaults to 240", () => {
const chunkSizeArg = lambdaCommand.args
? Reflect.get(lambdaCommand.args, "chunk-size")
: undefined;
expect(chunkSizeArg?.description).not.toContain("240");
expect(chunkSizeArg?.description).toContain("timeout");
});
});
+7 -1
View File
@@ -124,7 +124,11 @@ export async function runRender(args: RenderArgs): Promise<void> {
}
}
const config: SerializableDistributedRenderConfig = buildLambdaRenderConfig(args, variables);
const config: SerializableDistributedRenderConfig = buildLambdaRenderConfig(
args,
variables,
stack.reservedConcurrency,
);
// When the caller passes only --site-id, synthesise the minimum-shape
// SiteHandle pointing at the deterministic content-addressed key. The
@@ -241,6 +245,7 @@ function sleep(ms: number): Promise<void> {
export function buildLambdaRenderConfig(
args: RenderArgs,
variables: Record<string, unknown> | undefined,
workerConcurrency?: number,
): SerializableDistributedRenderConfig {
const config: SerializableDistributedRenderConfig = {
fps: args.fps,
@@ -253,6 +258,7 @@ export function buildLambdaRenderConfig(
chunkSize: args.chunkSize,
maxParallelChunks: args.maxParallelChunks,
targetChunkFrames: args.targetChunkFrames,
workerConcurrency,
runtimeCap: "lambda",
variables,
};
@@ -23,6 +23,7 @@ export interface StackOutputs {
functionName: string;
/** Lambda memory MB used during deploy; carried so cost math doesn't have to re-derive it. */
lambdaMemoryMb: number;
reservedConcurrency?: number;
deployedAt: string;
}