fix(producer): drop empty trailing chunk slice in distributed render plan (#1133)

resolveChunkPlan caps chunkCount at maxParallelChunks from the naive
count, then rounds effectiveChunkSize up to ceil(totalFrames /
chunkCount). When that ceil rounds up, the first (chunkCount - 1) chunks
can already cover every frame, so buildChunkSlices emits a final slice
with startFrame >= totalFrames — an empty [n, n) or inverted range.
renderChunk rejects it (framesInChunk <= 0) and, under Step Functions
retries, fails the whole distributed render even though [0, totalFrames)
is fully covered.

This is reachable from the user-facing CLI: `hyperframes lambda render
--chunk-size 10 --max-parallel-chunks 12` on a ~4s/30fps (121-frame)
composition yields chunkCount=12, effectiveChunkSize=11, and a 12th slice
of [121, 121).

Tighten chunkCount to ceil(totalFrames / effectiveChunkSize) after the
size is finalized, so the union stays exactly [0, totalFrames) with no
empty tail. This only lowers chunkCount in the explicit-small-chunkSize
case; the auto-sized and large-chunkSize paths already satisfy
ceil(totalFrames / effectiveChunkSize) >= chunkCount, so it's a no-op
there (existing tests' chunkCount values are unchanged).

Adds a regression test for the 121/10/12 case plus a grid property test
asserting contiguous, non-empty, exact coverage across explicit sizes.

Co-authored-by: Carlos Alcaraz <193642530+calcarazgre646@users.noreply.github.com>
This commit is contained in:
Carlos Alcaraz Gregor
2026-05-30 09:11:29 -04:00
committed by GitHub
co-authored by Carlos Alcaraz
parent 0e895cbff7
commit 81521a9a97
2 changed files with 49 additions and 1 deletions
@@ -125,6 +125,43 @@ describe("resolveChunkPlan", () => {
expect(result.chunkCount).toBe(5); expect(result.chunkCount).toBe(5);
expect(result.effectiveChunkSize).toBe(MIN_CHUNK_SIZE); expect(result.effectiveChunkSize).toBe(MIN_CHUNK_SIZE);
}); });
it("tightens chunkCount so an explicit small chunkSize leaves no empty trailing slice", () => {
// 121 frames, chunkSize=10, maxParallelChunks=12: ceil(121/10)=13 naive →
// capped at 12, but effectiveChunkSize rounds up to ceil(121/12)=11. Eleven
// 11-frame chunks already cover [0,121), so a 12th would be the empty slice
// [121,121) that renderChunk rejects (framesInChunk <= 0). chunkCount must
// tighten to 11.
const result = resolveChunkPlan(121, 10, 12);
expect(result.effectiveChunkSize).toBe(11);
expect(result.chunkCount).toBe(11);
const slices = buildChunkSlices(121, result.chunkCount, result.effectiveChunkSize);
expect(slices).toHaveLength(11);
expect(slices[slices.length - 1]).toEqual({ index: 10, startFrame: 110, endFrame: 121 });
});
it("never emits an empty or inverted slice across a grid of explicit chunk sizes", () => {
for (const totalFrames of [37, 50, 121, 333, 1000, 4001]) {
for (const maxParallel of [2, 4, 12, 16, 40]) {
for (const chunkSize of [10, 11, 15, 24, 100]) {
const { chunkCount, effectiveChunkSize } = resolveChunkPlan(
totalFrames,
chunkSize,
maxParallel,
);
expect(chunkCount).toBeLessThanOrEqual(maxParallel);
const slices = buildChunkSlices(totalFrames, chunkCount, effectiveChunkSize);
let cursor = 0;
for (const s of slices) {
expect(s.startFrame).toBe(cursor); // contiguous from 0
expect(s.endFrame).toBeGreaterThan(s.startFrame); // non-empty
cursor = s.endFrame;
}
expect(cursor).toBe(totalFrames); // union is exactly [0, totalFrames)
}
}
}
});
}); });
describe("buildChunkSlices", () => { describe("buildChunkSlices", () => {
@@ -400,6 +400,7 @@ export function measurePlanDirBytes(planDir: string): number {
* resolvedChunkSize = configChunkSize ?? max(MIN_CHUNK_SIZE, ceil(totalFrames / maxParallelChunks)) * resolvedChunkSize = configChunkSize ?? max(MIN_CHUNK_SIZE, ceil(totalFrames / maxParallelChunks))
* chunkCount = min(maxParallelChunks, ceil(totalFrames / resolvedChunkSize)) * chunkCount = min(maxParallelChunks, ceil(totalFrames / resolvedChunkSize))
* effectiveChunkSize = max(resolvedChunkSize, ceil(totalFrames / chunkCount)) * effectiveChunkSize = max(resolvedChunkSize, ceil(totalFrames / chunkCount))
* chunkCount = min(chunkCount, ceil(totalFrames / effectiveChunkSize)) // drop empty trailing slice
* *
* Long renders auto-rescale to fewer-but-longer chunks rather than * Long renders auto-rescale to fewer-but-longer chunks rather than
* fragmenting infinitely. Returned `chunkCount >= 1` (`totalFrames === 0` * fragmenting infinitely. Returned `chunkCount >= 1` (`totalFrames === 0`
@@ -434,7 +435,17 @@ export function resolveChunkPlan(
const naiveCount = Math.ceil(totalFrames / resolvedChunkSize); const naiveCount = Math.ceil(totalFrames / resolvedChunkSize);
const chunkCount = Math.min(maxParallelChunks, Math.max(1, naiveCount)); const chunkCount = Math.min(maxParallelChunks, Math.max(1, naiveCount));
const effectiveChunkSize = Math.max(resolvedChunkSize, Math.ceil(totalFrames / chunkCount)); const effectiveChunkSize = Math.max(resolvedChunkSize, Math.ceil(totalFrames / chunkCount));
return { chunkCount, effectiveChunkSize }; // Rounding effectiveChunkSize up can let the first (chunkCount - 1) chunks
// already cover every frame, leaving an empty/inverted trailing slice that
// buildChunkSlices would still emit and renderChunk would then reject
// (framesInChunk <= 0), failing the whole distributed render. Tighten
// chunkCount to the number of chunks effectiveChunkSize actually needs so the
// union stays exactly [0, totalFrames) with no empty tail. This only ever
// lowers chunkCount in the explicit-small-chunkSize case; the auto-sized and
// large-chunkSize paths already satisfy ceil(totalFrames / effectiveChunkSize)
// >= chunkCount, so it's a no-op there.
const tightChunkCount = Math.min(chunkCount, Math.ceil(totalFrames / effectiveChunkSize));
return { chunkCount: tightChunkCount, effectiveChunkSize };
} }
function assertPositiveInteger(name: string, value: number): void { function assertPositiveInteger(name: string, value: number): void {