From 730635d4e61d9d76140fae0a85baa936a5a7edb0 Mon Sep 17 00:00:00 2001 From: Miguel Angel Simon Sierra Date: Sat, 17 Oct 2026 05:52:07 -0700 Subject: [PATCH] fix(aws-lambda): keep render --wait working at hundreds of chunks Progress polls read the chunk Map Run's counts (one DescribeMapRun) instead of every child execution's history, take exact chunk frames and cost from the Map output once it exits, and read failed children only when the counts show a failure. The SDK client retries throttled calls up to 8 times with backoff. --- examples/aws-lambda/template.yaml | 4 + .../src/cdk/HyperframesRenderStack.ts | 6 +- .../src/sdk/getRenderProgress.test.ts | 194 ++++++++++++++---- .../aws-lambda/src/sdk/getRenderProgress.ts | 129 ++++++++---- packages/cli/src/commands/lambda/policies.ts | 1 + 5 files changed, 257 insertions(+), 77 deletions(-) diff --git a/examples/aws-lambda/template.yaml b/examples/aws-lambda/template.yaml index ab4f530b5d..5e4961aede 100644 --- a/examples/aws-lambda/template.yaml +++ b/examples/aws-lambda/template.yaml @@ -421,6 +421,8 @@ Resources: ChunkS3Uri.$: "$.Payload.ChunkS3Uri" ChunkIndex.$: "$.Payload.ChunkIndex" Sha256.$: "$.Payload.Sha256" + FramesEncoded.$: "$.Payload.FramesEncoded" + DurationMs.$: "$.Payload.DurationMs" Retry: - ErrorEquals: - FFMPEG_VERSION_MISMATCH @@ -531,6 +533,8 @@ Resources: ChunkS3Uri.$: "$.Payload.ChunkS3Uri" ChunkIndex.$: "$.Payload.ChunkIndex" Sha256.$: "$.Payload.Sha256" + FramesEncoded.$: "$.Payload.FramesEncoded" + DurationMs.$: "$.Payload.DurationMs" Retry: - ErrorEquals: - FFMPEG_VERSION_MISMATCH diff --git a/packages/aws-lambda/src/cdk/HyperframesRenderStack.ts b/packages/aws-lambda/src/cdk/HyperframesRenderStack.ts index 00665b8580..da7ee4b5a0 100644 --- a/packages/aws-lambda/src/cdk/HyperframesRenderStack.ts +++ b/packages/aws-lambda/src/cdk/HyperframesRenderStack.ts @@ -345,6 +345,8 @@ export class HyperframesRenderStack extends Construct { "ChunkS3Uri.$": "$.Payload.ChunkS3Uri", "ChunkIndex.$": "$.Payload.ChunkIndex", "Sha256.$": "$.Payload.Sha256", + "FramesEncoded.$": "$.Payload.FramesEncoded", + "DurationMs.$": "$.Payload.DurationMs", }, }); renderChunkTask.addRetry({ @@ -362,7 +364,7 @@ export class HyperframesRenderStack extends Construct { const renderChunks = new sfn.DistributedMap(this, "RenderChunks", { label: "RenderChunks", - // STANDARD children: EXPRESS ones stop at 5 minutes and a chunk may run the Lambda's full 15. + // Standard-type children: express ones stop at 5 minutes and a chunk may run the Lambda's full 15. mapExecutionType: sfn.StateMachineType.STANDARD, itemsPath: "$.Iterator.ChunkIndexes", itemSelector: { @@ -424,6 +426,8 @@ export class HyperframesRenderStack extends Construct { "ChunkS3Uri.$": "$.Payload.ChunkS3Uri", "ChunkIndex.$": "$.Payload.ChunkIndex", "Sha256.$": "$.Payload.Sha256", + "FramesEncoded.$": "$.Payload.FramesEncoded", + "DurationMs.$": "$.Payload.DurationMs", }, }); renderChunkV2Task.addRetry({ errors: NON_RETRYABLE_CHUNK, maxAttempts: 0 }); diff --git a/packages/aws-lambda/src/sdk/getRenderProgress.test.ts b/packages/aws-lambda/src/sdk/getRenderProgress.test.ts index ad32fede5d..2a6e93bd4c 100644 --- a/packages/aws-lambda/src/sdk/getRenderProgress.test.ts +++ b/packages/aws-lambda/src/sdk/getRenderProgress.test.ts @@ -6,7 +6,7 @@ import { type SFNClient, } from "@aws-sdk/client-sfn"; import { computeRenderCost } from "./costAccounting.js"; -import { getRenderProgress } from "./getRenderProgress.js"; +import { createProgressSfnClient, getRenderProgress } from "./getRenderProgress.js"; interface DescribeShape { status?: string; @@ -17,19 +17,30 @@ interface DescribeShape { interface FakeInput { executionArn?: string; mapRunArn?: string; + statusFilter?: string; nextToken?: string; } +interface FakeChild { + status: string; + history: HistoryEvent[]; +} + class FakeSFN { describe: DescribeShape = {}; // Pages of history events; FakeSFN walks them in order, paginating with `nextToken`. historyPages: HistoryEvent[][] = []; - // Distributed Map child executions: map run ARN -> child ARNs, child ARN -> its history. - mapRunChildren: Record = {}; - childHistories: Record = {}; + // Distributed Map: map run ARN -> item counts and child executions (child ARN -> status + history). + mapRunCounts: Record> = {}; + mapRunChildren: Record> = {}; + calls: string[] = []; async send(command: unknown): Promise { const { constructor, input } = command as { constructor: { name: string }; input: FakeInput }; + this.calls.push(constructor.name); if (constructor.name === "DescribeExecutionCommand") return this.describeResult(); + if (constructor.name === "DescribeMapRunCommand") { + return { itemCounts: this.mapRunCounts[input.mapRunArn ?? ""] }; + } if (constructor.name === "ListExecutionsCommand") return this.listResult(input); if (constructor.name === "GetExecutionHistoryCommand") return this.historyResult(input); throw new Error(`FakeSFN: unexpected command ${constructor.name}`); @@ -41,13 +52,16 @@ class FakeSFN { stopDate: this.describe.stopDate, }; } - private listResult({ mapRunArn = "" }: FakeInput): unknown { - const arns = this.mapRunChildren[mapRunArn] ?? []; - return { executions: arns.map((executionArn) => ({ executionArn })) }; + private listResult({ mapRunArn = "", statusFilter }: FakeInput): unknown { + const children = Object.entries(this.mapRunChildren[mapRunArn] ?? {}); + const matching = children.filter(([, c]) => !statusFilter || c.status === statusFilter); + return { executions: matching.map(([executionArn]) => ({ executionArn })) }; } private historyResult({ executionArn = "", nextToken }: FakeInput): unknown { - const child = this.childHistories[executionArn]; - return historyPage(child ? [child] : this.historyPages, Number(nextToken ?? 0)); + const child = Object.values(this.mapRunChildren) + .map((children) => children[executionArn]) + .find(Boolean); + return historyPage(child ? [child.history] : this.historyPages, Number(nextToken ?? 0)); } } @@ -310,14 +324,15 @@ describe("getRenderProgress", () => { expect(progress.overallProgress).toBe(1); }); - it("reads chunk frames, cost and errors from Distributed Map child executions", async () => { - const sfn = new FakeSFN(); + describe("Distributed Map chunks", () => { const mapRunArn = "arn:aws:states:us-east-2:0:mapRun:hf-render/RenderChunksV2:run"; - sfn.historyPages = [ - [ + const CHUNKS = 225; + + function parentUntilMapRun(totalFrames: number): HistoryEvent[] { + return [ stateEntered("PlanV2"), taskScheduled(), - taskSucceeded({ TotalFrames: 300, DurationMs: 1000 }), + taskSucceeded({ TotalFrames: totalFrames, DurationMs: 1000 }), { type: "MapStateEntered", id: 1, @@ -330,34 +345,133 @@ describe("getRenderProgress", () => { timestamp: new Date(), mapRunStartedEventDetails: { mapRunArn }, } as HistoryEvent, - ], - ]; - sfn.mapRunChildren[mapRunArn] = ["child-0", "child-1"]; - sfn.childHistories["child-0"] = [ - stateEntered("RenderChunkV2"), - taskScheduled(), - taskSucceeded({ FramesEncoded: 150, DurationMs: 4000 }), - ]; - sfn.childHistories["child-1"] = [ - stateEntered("RenderChunkV2"), - taskScheduled(), - taskFailed("PLAN_HASH_MISMATCH", "hash drift"), - ]; - const p = await getRenderProgress({ executionArn: "arn:x", sfn: sfn as unknown as SFNClient }); - expect(p.framesRendered).toBe(150); - expect(p.overallProgress).toBeCloseTo(0.1 + 0.8 * 0.5, 5); - expect(p.lambdasInvoked).toBe(3); - const billed = [1000, 4000].map((billedDurationMs) => ({ - billedDurationMs, - memorySizeMb: 10240, - estimated: false, - })); - expect(p.costs.breakdown.lambdaUsd).toBe(computeRenderCost(billed, 0).breakdown.lambdaUsd); - expect(p.errors).toEqual([ - { state: "RenderChunkV2", error: "PLAN_HASH_MISMATCH", cause: "hash drift" }, - ]); + ]; + } + + function chunkChild(status: string, last: HistoryEvent): FakeChild { + return { status, history: [stateEntered("RenderChunkV2"), taskScheduled(), last] }; + } + + // Children with real histories, so a per-child reader would find (and pay for) all of them. + function fakeWithChildren(statuses: string[]): FakeSFN { + const sfn = new FakeSFN(); + sfn.historyPages = [parentUntilMapRun(3600)]; + sfn.mapRunChildren[mapRunArn] = Object.fromEntries( + statuses.map((status, i) => [ + `child-${i}`, + status === "FAILED" + ? chunkChild(status, taskFailed("PLAN_HASH_MISMATCH", `chunk ${i} hash drift`)) + : chunkChild(status, taskSucceeded({ FramesEncoded: 16, DurationMs: 4000 })), + ]), + ); + return sfn; + } + + it("polls 225 running chunks with a constant number of calls", async () => { + const statuses = Array.from({ length: CHUNKS }, (_, i) => + i < 125 ? "SUCCEEDED" : "RUNNING", + ); + const sfn = fakeWithChildren(statuses); + sfn.mapRunCounts[mapRunArn] = { total: CHUNKS, pending: 0, running: 100, succeeded: 125 }; + const p = await getRenderProgress({ + executionArn: "arn:x", + sfn: sfn as unknown as SFNClient, + }); + expect(sfn.calls).toEqual([ + "DescribeExecutionCommand", + "GetExecutionHistoryCommand", + "DescribeMapRunCommand", + ]); + expect(p.framesRendered).toBe(Math.floor((3600 * 125) / CHUNKS)); + expect(p.overallProgress).toBeCloseTo(0.1 + 0.8 * (125 / CHUNKS), 5); + expect(p.lambdasInvoked).toBe(1 + CHUNKS); + expect(p.errors).toEqual([]); + }); + + it("reads exact chunk frames and cost from the Map output once it exits", async () => { + const sfn = new FakeSFN(); + const chunks = Array.from({ length: CHUNKS }, (_, i) => ({ + ChunkIndex: i, + FramesEncoded: 16, + DurationMs: 4000, + })); + sfn.historyPages = [ + [ + ...parentUntilMapRun(3600), + { + type: "MapStateExited", + id: 1, + timestamp: new Date(), + stateExitedEventDetails: { + name: "RenderChunksV2", + output: JSON.stringify({ Chunks: chunks }), + }, + } as HistoryEvent, + ], + ]; + const p = await getRenderProgress({ + executionArn: "arn:x", + sfn: sfn as unknown as SFNClient, + }); + expect(sfn.calls).toEqual(["DescribeExecutionCommand", "GetExecutionHistoryCommand"]); + expect(p.framesRendered).toBe(3600); + expect(p.lambdasInvoked).toBe(1 + CHUNKS); + const billed = [1000, ...chunks.map((c) => c.DurationMs)].map((billedDurationMs) => ({ + billedDurationMs, + memorySizeMb: 10240, + estimated: false, + })); + expect(p.costs.breakdown.lambdaUsd).toBe(computeRenderCost(billed, 0).breakdown.lambdaUsd); + }); + + it("surfaces a failed chunk's own error, reading only failed children", async () => { + const statuses = Array.from({ length: CHUNKS }, (_, i) => + i === 7 ? "FAILED" : i < 100 ? "SUCCEEDED" : "ABORTED", + ); + const sfn = fakeWithChildren(statuses); + sfn.mapRunCounts[mapRunArn] = { total: CHUNKS, succeeded: 99, failed: 1, aborted: 125 }; + const p = await getRenderProgress({ + executionArn: "arn:x", + sfn: sfn as unknown as SFNClient, + }); + expect(p.errors).toEqual([ + { state: "RenderChunkV2", error: "PLAN_HASH_MISMATCH", cause: "chunk 7 hash drift" }, + ]); + expect(sfn.calls).toHaveLength(5); + }); }); + it("backs off on ThrottlingException and still returns progress", async () => { + let throttlesLeft = 3; + const targets: string[] = []; + const json = (statusCode: number, body: unknown) => ({ + response: { + statusCode, + headers: { "content-type": "application/x-amz-json-1.0" }, + body: new TextEncoder().encode(JSON.stringify(body)), + }, + }); + const sfn = createProgressSfnClient({ + region: "us-east-2", + credentials: { accessKeyId: "a", secretAccessKey: "b" }, + requestHandler: { + async handle(req: { headers: Record }) { + targets.push(req.headers["x-amz-target"] ?? ""); + if (throttlesLeft-- > 0) { + return json(400, { __type: "ThrottlingException", message: "Rate exceeded" }); + } + if (req.headers["x-amz-target"]?.endsWith("DescribeExecution")) { + return json(200, { status: "RUNNING", startDate: 1_760_000_000 }); + } + return json(200, { events: [] }); + }, + }, + }); + const p = await getRenderProgress({ executionArn: "arn:x", sfn }); + expect(p.status).toBe("RUNNING"); + expect(targets).toHaveLength(5); + }, 20_000); + it("requires executionArn", async () => { await expect(getRenderProgress({ executionArn: "" })).rejects.toThrow(/executionArn/); }); diff --git a/packages/aws-lambda/src/sdk/getRenderProgress.ts b/packages/aws-lambda/src/sdk/getRenderProgress.ts index a01e4101ca..881cd13b7a 100644 --- a/packages/aws-lambda/src/sdk/getRenderProgress.ts +++ b/packages/aws-lambda/src/sdk/getRenderProgress.ts @@ -2,11 +2,10 @@ * `getRenderProgress` — read-only progress + cost snapshot for a single * render started by {@link renderToLambda}. * - * Pulls one `DescribeExecution` + one `GetExecutionHistory` per call, plus - * one history per chunk: the Distributed Map runs each chunk as a child - * execution whose events never reach the parent history. Histories are - * paginated server-side; the helper loops until exhausted so a 1,000-event - * Step Functions execution still produces a single `RenderProgress` snapshot. + * Pulls one `DescribeExecution` + one `GetExecutionHistory` per call. While + * the chunk Distributed Map runs, one `DescribeMapRun` adds its item counts; + * once it exits, chunk frames and cost come from the Map's output. Failed + * chunks' histories are read only when the counts show a failure. * * Progress math: * - 0 before Plan completes (no frame count is known yet) @@ -23,10 +22,12 @@ import { DescribeExecutionCommand, + DescribeMapRunCommand, GetExecutionHistoryCommand, type HistoryEvent, ListExecutionsCommand, SFNClient, + type SFNClientConfig, } from "@aws-sdk/client-sfn"; import { type BilledLambdaInvocation, @@ -89,12 +90,17 @@ export interface RenderProgress { const DEFAULT_MEMORY_MB = 10240; +/** Rides out `ThrottlingException` with the SDK's backoff instead of throwing after the default 3 tries. */ +export function createProgressSfnClient(config: SFNClientConfig = {}): SFNClient { + return new SFNClient({ maxAttempts: 8, ...config }); +} + /** Pull a current progress snapshot for one render. */ export async function getRenderProgress(opts: GetRenderProgressOptions): Promise { if (!opts.executionArn) { throw new Error("[getRenderProgress] executionArn is required"); } - const sfn = opts.sfn ?? new SFNClient({ region: opts.region }); + const sfn = opts.sfn ?? createProgressSfnClient({ region: opts.region }); const memoryMb = opts.defaultMemorySizeMb ?? DEFAULT_MEMORY_MB; const describe = await sfn.send( @@ -105,11 +111,10 @@ export async function getRenderProgress(opts: GetRenderProgressOptions): Promise const endedAt = describe.stopDate?.toISOString() ?? null; const history = await loadFullHistory(sfn, opts.executionArn); - // Each child history opens with its own state entry, so one pass over the concatenation is safe. - const summary = summarizeHistory( - [...history, ...(await loadChunkHistories(sfn, history))], - memoryMb, - ); + const summary = summarizeHistory(history, memoryMb); + const mapRun = findChunkMapRun(history); + if (mapRun?.exited) addChunkResults(summary, mapRun.output, memoryMb); + else if (mapRun) await addRunningMapRun(sfn, mapRun.arn, summary, memoryMb); const costs = computeRenderCost(summary.lambdaInvocations, summary.stateTransitions); const overallProgress = computeOverallProgress({ @@ -153,34 +158,82 @@ async function loadFullHistory(sfn: SFNClient, executionArn: string): Promise { - const mapRunArns = parent.flatMap((ev) => ev.mapRunStartedEventDetails?.mapRunArn ?? []); - const childArns = ( - await Promise.all(mapRunArns.map((arn) => listMapRunChildren(sfn, arn))) - ).flat(); - const histories: HistoryEvent[][] = []; - for (let i = 0; i < childArns.length; i += CHILD_HISTORY_READS_IN_FLIGHT) { - const batch = childArns.slice(i, i + CHILD_HISTORY_READS_IN_FLIGHT); - histories.push(...(await Promise.all(batch.map((arn) => loadFullHistory(sfn, arn))))); - } - return histories.flat(); +interface ChunkMapRun { + arn: string; + exited: boolean; + output: string | undefined; } -async function listMapRunChildren(sfn: SFNClient, mapRunArn: string): Promise { - const arns: string[] = []; - let nextToken: string | undefined; - do { - const res = await sfn.send( - new ListExecutionsCommand({ mapRunArn, maxResults: 1000, nextToken }), - ); - arns.push(...(res.executions ?? []).flatMap((child) => child.executionArn ?? [])); - nextToken = res.nextToken; - } while (nextToken); - return arns; +/** The chunk Distributed Map's run; `null` for an inline Map, whose chunk events are in the parent history. */ +function findChunkMapRun(events: HistoryEvent[]): ChunkMapRun | null { + let run: ChunkMapRun | null = null; + for (const ev of events) { + const arn = ev.mapRunStartedEventDetails?.mapRunArn; + if (arn) run = { arn, exited: false, output: undefined }; + const exit = ev.stateExitedEventDetails; + if (run && ev.type === "MapStateExited" && isRenderChunksMapState(exit?.name)) { + run = { ...run, exited: true, output: exit?.output }; + } + } + return run; +} + +/** Exact chunk frames and cost from the chunk results the Map's output carries. */ +function addChunkResults(summary: HistorySummary, mapOutput: string | undefined, memoryMb: number) { + const chunks = (parseJson(mapOutput) as { Chunks?: unknown } | null)?.Chunks; + for (const chunk of Array.isArray(chunks) ? chunks : []) { + summary.lambdasInvoked++; + summary.stateTransitions++; + const billedDurationMs = inferBilledMs(chunk); + summary.lambdaInvocations.push({ + billedDurationMs, + memorySizeMb: memoryMb, + estimated: billedDurationMs === 0, + }); + applyPayloadFrameCounts(chunk, "RenderChunk", (delta) => { + summary.framesRendered += delta; + }); + } +} + +// Open Map Run: frames estimated from the succeeded share; chunk Lambda cost waits for the Map's exit. +async function addRunningMapRun( + sfn: SFNClient, + mapRunArn: string, + summary: HistorySummary, + memoryMb: number, +): Promise { + const res = await sfn.send(new DescribeMapRunCommand({ mapRunArn })); + const { total = 0, pending = 0, succeeded = 0, failed = 0 } = res.itemCounts ?? {}; + summary.lambdasInvoked += total - pending; + summary.stateTransitions += total - pending; + if (summary.totalFrames !== null && total > 0) { + summary.framesRendered += Math.floor((summary.totalFrames * succeeded) / total); + } + if (failed > 0) summary.errors.push(...(await failedChunkErrors(sfn, mapRunArn, memoryMb))); +} + +async function failedChunkErrors( + sfn: SFNClient, + mapRunArn: string, + memoryMb: number, +): Promise { + const failed = await sfn.send( + new ListExecutionsCommand({ + mapRunArn, + statusFilter: "FAILED", + maxResults: FAILED_CHUNK_HISTORIES_READ, + }), + ); + const errors: RenderError[] = []; + for (const child of failed.executions ?? []) { + if (!child.executionArn) continue; + const childHistory = await loadFullHistory(sfn, child.executionArn); + errors.push(...summarizeHistory(childHistory, memoryMb).errors); + } + return errors; } interface HistorySummary { @@ -396,6 +449,10 @@ function isRenderChunkState(name: string | null | undefined): boolean { return name === "RenderChunk" || name === "RenderChunkV2"; } +function isRenderChunksMapState(name: string | null | undefined): boolean { + return name === "RenderChunks" || name === "RenderChunksV2"; +} + function isAssembleState(name: string | null | undefined): boolean { return name === "Assemble" || name === "AssembleV2"; } diff --git a/packages/cli/src/commands/lambda/policies.ts b/packages/cli/src/commands/lambda/policies.ts index e1f12d0790..72a3567e3d 100644 --- a/packages/cli/src/commands/lambda/policies.ts +++ b/packages/cli/src/commands/lambda/policies.ts @@ -144,6 +144,7 @@ const REQUIRED_ACTIONS = { "states:CreateStateMachine", "states:DeleteStateMachine", "states:DescribeExecution", + "states:DescribeMapRun", "states:DescribeStateMachine", "states:GetExecutionHistory", "states:ListExecutions",