diff --git a/examples/aws-lambda/template.yaml b/examples/aws-lambda/template.yaml index ab4f530b5d..5e4961aede 100644 --- a/examples/aws-lambda/template.yaml +++ b/examples/aws-lambda/template.yaml @@ -421,6 +421,8 @@ Resources: ChunkS3Uri.$: "$.Payload.ChunkS3Uri" ChunkIndex.$: "$.Payload.ChunkIndex" Sha256.$: "$.Payload.Sha256" + FramesEncoded.$: "$.Payload.FramesEncoded" + DurationMs.$: "$.Payload.DurationMs" Retry: - ErrorEquals: - FFMPEG_VERSION_MISMATCH @@ -531,6 +533,8 @@ Resources: ChunkS3Uri.$: "$.Payload.ChunkS3Uri" ChunkIndex.$: "$.Payload.ChunkIndex" Sha256.$: "$.Payload.Sha256" + FramesEncoded.$: "$.Payload.FramesEncoded" + DurationMs.$: "$.Payload.DurationMs" Retry: - ErrorEquals: - FFMPEG_VERSION_MISMATCH diff --git a/packages/aws-lambda/src/cdk/HyperframesRenderStack.ts b/packages/aws-lambda/src/cdk/HyperframesRenderStack.ts index 00665b8580..da7ee4b5a0 100644 --- a/packages/aws-lambda/src/cdk/HyperframesRenderStack.ts +++ b/packages/aws-lambda/src/cdk/HyperframesRenderStack.ts @@ -345,6 +345,8 @@ export class HyperframesRenderStack extends Construct { "ChunkS3Uri.$": "$.Payload.ChunkS3Uri", "ChunkIndex.$": "$.Payload.ChunkIndex", "Sha256.$": "$.Payload.Sha256", + "FramesEncoded.$": "$.Payload.FramesEncoded", + "DurationMs.$": "$.Payload.DurationMs", }, }); renderChunkTask.addRetry({ @@ -362,7 +364,7 @@ export class HyperframesRenderStack extends Construct { const renderChunks = new sfn.DistributedMap(this, "RenderChunks", { label: "RenderChunks", - // STANDARD children: EXPRESS ones stop at 5 minutes and a chunk may run the Lambda's full 15. + // Standard-type children: express ones stop at 5 minutes and a chunk may run the Lambda's full 15. mapExecutionType: sfn.StateMachineType.STANDARD, itemsPath: "$.Iterator.ChunkIndexes", itemSelector: { @@ -424,6 +426,8 @@ export class HyperframesRenderStack extends Construct { "ChunkS3Uri.$": "$.Payload.ChunkS3Uri", "ChunkIndex.$": "$.Payload.ChunkIndex", "Sha256.$": "$.Payload.Sha256", + "FramesEncoded.$": "$.Payload.FramesEncoded", + "DurationMs.$": "$.Payload.DurationMs", }, }); renderChunkV2Task.addRetry({ errors: NON_RETRYABLE_CHUNK, maxAttempts: 0 }); diff --git a/packages/aws-lambda/src/sdk/getRenderProgress.test.ts b/packages/aws-lambda/src/sdk/getRenderProgress.test.ts index ad32fede5d..2a6e93bd4c 100644 --- a/packages/aws-lambda/src/sdk/getRenderProgress.test.ts +++ b/packages/aws-lambda/src/sdk/getRenderProgress.test.ts @@ -6,7 +6,7 @@ import { type SFNClient, } from "@aws-sdk/client-sfn"; import { computeRenderCost } from "./costAccounting.js"; -import { getRenderProgress } from "./getRenderProgress.js"; +import { createProgressSfnClient, getRenderProgress } from "./getRenderProgress.js"; interface DescribeShape { status?: string; @@ -17,19 +17,30 @@ interface DescribeShape { interface FakeInput { executionArn?: string; mapRunArn?: string; + statusFilter?: string; nextToken?: string; } +interface FakeChild { + status: string; + history: HistoryEvent[]; +} + class FakeSFN { describe: DescribeShape = {}; // Pages of history events; FakeSFN walks them in order, paginating with `nextToken`. historyPages: HistoryEvent[][] = []; - // Distributed Map child executions: map run ARN -> child ARNs, child ARN -> its history. - mapRunChildren: Record = {}; - childHistories: Record = {}; + // Distributed Map: map run ARN -> item counts and child executions (child ARN -> status + history). + mapRunCounts: Record> = {}; + mapRunChildren: Record> = {}; + calls: string[] = []; async send(command: unknown): Promise { const { constructor, input } = command as { constructor: { name: string }; input: FakeInput }; + this.calls.push(constructor.name); if (constructor.name === "DescribeExecutionCommand") return this.describeResult(); + if (constructor.name === "DescribeMapRunCommand") { + return { itemCounts: this.mapRunCounts[input.mapRunArn ?? ""] }; + } if (constructor.name === "ListExecutionsCommand") return this.listResult(input); if (constructor.name === "GetExecutionHistoryCommand") return this.historyResult(input); throw new Error(`FakeSFN: unexpected command ${constructor.name}`); @@ -41,13 +52,16 @@ class FakeSFN { stopDate: this.describe.stopDate, }; } - private listResult({ mapRunArn = "" }: FakeInput): unknown { - const arns = this.mapRunChildren[mapRunArn] ?? []; - return { executions: arns.map((executionArn) => ({ executionArn })) }; + private listResult({ mapRunArn = "", statusFilter }: FakeInput): unknown { + const children = Object.entries(this.mapRunChildren[mapRunArn] ?? {}); + const matching = children.filter(([, c]) => !statusFilter || c.status === statusFilter); + return { executions: matching.map(([executionArn]) => ({ executionArn })) }; } private historyResult({ executionArn = "", nextToken }: FakeInput): unknown { - const child = this.childHistories[executionArn]; - return historyPage(child ? [child] : this.historyPages, Number(nextToken ?? 0)); + const child = Object.values(this.mapRunChildren) + .map((children) => children[executionArn]) + .find(Boolean); + return historyPage(child ? [child.history] : this.historyPages, Number(nextToken ?? 0)); } } @@ -310,14 +324,15 @@ describe("getRenderProgress", () => { expect(progress.overallProgress).toBe(1); }); - it("reads chunk frames, cost and errors from Distributed Map child executions", async () => { - const sfn = new FakeSFN(); + describe("Distributed Map chunks", () => { const mapRunArn = "arn:aws:states:us-east-2:0:mapRun:hf-render/RenderChunksV2:run"; - sfn.historyPages = [ - [ + const CHUNKS = 225; + + function parentUntilMapRun(totalFrames: number): HistoryEvent[] { + return [ stateEntered("PlanV2"), taskScheduled(), - taskSucceeded({ TotalFrames: 300, DurationMs: 1000 }), + taskSucceeded({ TotalFrames: totalFrames, DurationMs: 1000 }), { type: "MapStateEntered", id: 1, @@ -330,34 +345,133 @@ describe("getRenderProgress", () => { timestamp: new Date(), mapRunStartedEventDetails: { mapRunArn }, } as HistoryEvent, - ], - ]; - sfn.mapRunChildren[mapRunArn] = ["child-0", "child-1"]; - sfn.childHistories["child-0"] = [ - stateEntered("RenderChunkV2"), - taskScheduled(), - taskSucceeded({ FramesEncoded: 150, DurationMs: 4000 }), - ]; - sfn.childHistories["child-1"] = [ - stateEntered("RenderChunkV2"), - taskScheduled(), - taskFailed("PLAN_HASH_MISMATCH", "hash drift"), - ]; - const p = await getRenderProgress({ executionArn: "arn:x", sfn: sfn as unknown as SFNClient }); - expect(p.framesRendered).toBe(150); - expect(p.overallProgress).toBeCloseTo(0.1 + 0.8 * 0.5, 5); - expect(p.lambdasInvoked).toBe(3); - const billed = [1000, 4000].map((billedDurationMs) => ({ - billedDurationMs, - memorySizeMb: 10240, - estimated: false, - })); - expect(p.costs.breakdown.lambdaUsd).toBe(computeRenderCost(billed, 0).breakdown.lambdaUsd); - expect(p.errors).toEqual([ - { state: "RenderChunkV2", error: "PLAN_HASH_MISMATCH", cause: "hash drift" }, - ]); + ]; + } + + function chunkChild(status: string, last: HistoryEvent): FakeChild { + return { status, history: [stateEntered("RenderChunkV2"), taskScheduled(), last] }; + } + + // Children with real histories, so a per-child reader would find (and pay for) all of them. + function fakeWithChildren(statuses: string[]): FakeSFN { + const sfn = new FakeSFN(); + sfn.historyPages = [parentUntilMapRun(3600)]; + sfn.mapRunChildren[mapRunArn] = Object.fromEntries( + statuses.map((status, i) => [ + `child-${i}`, + status === "FAILED" + ? chunkChild(status, taskFailed("PLAN_HASH_MISMATCH", `chunk ${i} hash drift`)) + : chunkChild(status, taskSucceeded({ FramesEncoded: 16, DurationMs: 4000 })), + ]), + ); + return sfn; + } + + it("polls 225 running chunks with a constant number of calls", async () => { + const statuses = Array.from({ length: CHUNKS }, (_, i) => + i < 125 ? "SUCCEEDED" : "RUNNING", + ); + const sfn = fakeWithChildren(statuses); + sfn.mapRunCounts[mapRunArn] = { total: CHUNKS, pending: 0, running: 100, succeeded: 125 }; + const p = await getRenderProgress({ + executionArn: "arn:x", + sfn: sfn as unknown as SFNClient, + }); + expect(sfn.calls).toEqual([ + "DescribeExecutionCommand", + "GetExecutionHistoryCommand", + "DescribeMapRunCommand", + ]); + expect(p.framesRendered).toBe(Math.floor((3600 * 125) / CHUNKS)); + expect(p.overallProgress).toBeCloseTo(0.1 + 0.8 * (125 / CHUNKS), 5); + expect(p.lambdasInvoked).toBe(1 + CHUNKS); + expect(p.errors).toEqual([]); + }); + + it("reads exact chunk frames and cost from the Map output once it exits", async () => { + const sfn = new FakeSFN(); + const chunks = Array.from({ length: CHUNKS }, (_, i) => ({ + ChunkIndex: i, + FramesEncoded: 16, + DurationMs: 4000, + })); + sfn.historyPages = [ + [ + ...parentUntilMapRun(3600), + { + type: "MapStateExited", + id: 1, + timestamp: new Date(), + stateExitedEventDetails: { + name: "RenderChunksV2", + output: JSON.stringify({ Chunks: chunks }), + }, + } as HistoryEvent, + ], + ]; + const p = await getRenderProgress({ + executionArn: "arn:x", + sfn: sfn as unknown as SFNClient, + }); + expect(sfn.calls).toEqual(["DescribeExecutionCommand", "GetExecutionHistoryCommand"]); + expect(p.framesRendered).toBe(3600); + expect(p.lambdasInvoked).toBe(1 + CHUNKS); + const billed = [1000, ...chunks.map((c) => c.DurationMs)].map((billedDurationMs) => ({ + billedDurationMs, + memorySizeMb: 10240, + estimated: false, + })); + expect(p.costs.breakdown.lambdaUsd).toBe(computeRenderCost(billed, 0).breakdown.lambdaUsd); + }); + + it("surfaces a failed chunk's own error, reading only failed children", async () => { + const statuses = Array.from({ length: CHUNKS }, (_, i) => + i === 7 ? "FAILED" : i < 100 ? "SUCCEEDED" : "ABORTED", + ); + const sfn = fakeWithChildren(statuses); + sfn.mapRunCounts[mapRunArn] = { total: CHUNKS, succeeded: 99, failed: 1, aborted: 125 }; + const p = await getRenderProgress({ + executionArn: "arn:x", + sfn: sfn as unknown as SFNClient, + }); + expect(p.errors).toEqual([ + { state: "RenderChunkV2", error: "PLAN_HASH_MISMATCH", cause: "chunk 7 hash drift" }, + ]); + expect(sfn.calls).toHaveLength(5); + }); }); + it("backs off on ThrottlingException and still returns progress", async () => { + let throttlesLeft = 3; + const targets: string[] = []; + const json = (statusCode: number, body: unknown) => ({ + response: { + statusCode, + headers: { "content-type": "application/x-amz-json-1.0" }, + body: new TextEncoder().encode(JSON.stringify(body)), + }, + }); + const sfn = createProgressSfnClient({ + region: "us-east-2", + credentials: { accessKeyId: "a", secretAccessKey: "b" }, + requestHandler: { + async handle(req: { headers: Record }) { + targets.push(req.headers["x-amz-target"] ?? ""); + if (throttlesLeft-- > 0) { + return json(400, { __type: "ThrottlingException", message: "Rate exceeded" }); + } + if (req.headers["x-amz-target"]?.endsWith("DescribeExecution")) { + return json(200, { status: "RUNNING", startDate: 1_760_000_000 }); + } + return json(200, { events: [] }); + }, + }, + }); + const p = await getRenderProgress({ executionArn: "arn:x", sfn }); + expect(p.status).toBe("RUNNING"); + expect(targets).toHaveLength(5); + }, 20_000); + it("requires executionArn", async () => { await expect(getRenderProgress({ executionArn: "" })).rejects.toThrow(/executionArn/); }); diff --git a/packages/aws-lambda/src/sdk/getRenderProgress.ts b/packages/aws-lambda/src/sdk/getRenderProgress.ts index a01e4101ca..881cd13b7a 100644 --- a/packages/aws-lambda/src/sdk/getRenderProgress.ts +++ b/packages/aws-lambda/src/sdk/getRenderProgress.ts @@ -2,11 +2,10 @@ * `getRenderProgress` — read-only progress + cost snapshot for a single * render started by {@link renderToLambda}. * - * Pulls one `DescribeExecution` + one `GetExecutionHistory` per call, plus - * one history per chunk: the Distributed Map runs each chunk as a child - * execution whose events never reach the parent history. Histories are - * paginated server-side; the helper loops until exhausted so a 1,000-event - * Step Functions execution still produces a single `RenderProgress` snapshot. + * Pulls one `DescribeExecution` + one `GetExecutionHistory` per call. While + * the chunk Distributed Map runs, one `DescribeMapRun` adds its item counts; + * once it exits, chunk frames and cost come from the Map's output. Failed + * chunks' histories are read only when the counts show a failure. * * Progress math: * - 0 before Plan completes (no frame count is known yet) @@ -23,10 +22,12 @@ import { DescribeExecutionCommand, + DescribeMapRunCommand, GetExecutionHistoryCommand, type HistoryEvent, ListExecutionsCommand, SFNClient, + type SFNClientConfig, } from "@aws-sdk/client-sfn"; import { type BilledLambdaInvocation, @@ -89,12 +90,17 @@ export interface RenderProgress { const DEFAULT_MEMORY_MB = 10240; +/** Rides out `ThrottlingException` with the SDK's backoff instead of throwing after the default 3 tries. */ +export function createProgressSfnClient(config: SFNClientConfig = {}): SFNClient { + return new SFNClient({ maxAttempts: 8, ...config }); +} + /** Pull a current progress snapshot for one render. */ export async function getRenderProgress(opts: GetRenderProgressOptions): Promise { if (!opts.executionArn) { throw new Error("[getRenderProgress] executionArn is required"); } - const sfn = opts.sfn ?? new SFNClient({ region: opts.region }); + const sfn = opts.sfn ?? createProgressSfnClient({ region: opts.region }); const memoryMb = opts.defaultMemorySizeMb ?? DEFAULT_MEMORY_MB; const describe = await sfn.send( @@ -105,11 +111,10 @@ export async function getRenderProgress(opts: GetRenderProgressOptions): Promise const endedAt = describe.stopDate?.toISOString() ?? null; const history = await loadFullHistory(sfn, opts.executionArn); - // Each child history opens with its own state entry, so one pass over the concatenation is safe. - const summary = summarizeHistory( - [...history, ...(await loadChunkHistories(sfn, history))], - memoryMb, - ); + const summary = summarizeHistory(history, memoryMb); + const mapRun = findChunkMapRun(history); + if (mapRun?.exited) addChunkResults(summary, mapRun.output, memoryMb); + else if (mapRun) await addRunningMapRun(sfn, mapRun.arn, summary, memoryMb); const costs = computeRenderCost(summary.lambdaInvocations, summary.stateTransitions); const overallProgress = computeOverallProgress({ @@ -153,34 +158,82 @@ async function loadFullHistory(sfn: SFNClient, executionArn: string): Promise { - const mapRunArns = parent.flatMap((ev) => ev.mapRunStartedEventDetails?.mapRunArn ?? []); - const childArns = ( - await Promise.all(mapRunArns.map((arn) => listMapRunChildren(sfn, arn))) - ).flat(); - const histories: HistoryEvent[][] = []; - for (let i = 0; i < childArns.length; i += CHILD_HISTORY_READS_IN_FLIGHT) { - const batch = childArns.slice(i, i + CHILD_HISTORY_READS_IN_FLIGHT); - histories.push(...(await Promise.all(batch.map((arn) => loadFullHistory(sfn, arn))))); - } - return histories.flat(); +interface ChunkMapRun { + arn: string; + exited: boolean; + output: string | undefined; } -async function listMapRunChildren(sfn: SFNClient, mapRunArn: string): Promise { - const arns: string[] = []; - let nextToken: string | undefined; - do { - const res = await sfn.send( - new ListExecutionsCommand({ mapRunArn, maxResults: 1000, nextToken }), - ); - arns.push(...(res.executions ?? []).flatMap((child) => child.executionArn ?? [])); - nextToken = res.nextToken; - } while (nextToken); - return arns; +/** The chunk Distributed Map's run; `null` for an inline Map, whose chunk events are in the parent history. */ +function findChunkMapRun(events: HistoryEvent[]): ChunkMapRun | null { + let run: ChunkMapRun | null = null; + for (const ev of events) { + const arn = ev.mapRunStartedEventDetails?.mapRunArn; + if (arn) run = { arn, exited: false, output: undefined }; + const exit = ev.stateExitedEventDetails; + if (run && ev.type === "MapStateExited" && isRenderChunksMapState(exit?.name)) { + run = { ...run, exited: true, output: exit?.output }; + } + } + return run; +} + +/** Exact chunk frames and cost from the chunk results the Map's output carries. */ +function addChunkResults(summary: HistorySummary, mapOutput: string | undefined, memoryMb: number) { + const chunks = (parseJson(mapOutput) as { Chunks?: unknown } | null)?.Chunks; + for (const chunk of Array.isArray(chunks) ? chunks : []) { + summary.lambdasInvoked++; + summary.stateTransitions++; + const billedDurationMs = inferBilledMs(chunk); + summary.lambdaInvocations.push({ + billedDurationMs, + memorySizeMb: memoryMb, + estimated: billedDurationMs === 0, + }); + applyPayloadFrameCounts(chunk, "RenderChunk", (delta) => { + summary.framesRendered += delta; + }); + } +} + +// Open Map Run: frames estimated from the succeeded share; chunk Lambda cost waits for the Map's exit. +async function addRunningMapRun( + sfn: SFNClient, + mapRunArn: string, + summary: HistorySummary, + memoryMb: number, +): Promise { + const res = await sfn.send(new DescribeMapRunCommand({ mapRunArn })); + const { total = 0, pending = 0, succeeded = 0, failed = 0 } = res.itemCounts ?? {}; + summary.lambdasInvoked += total - pending; + summary.stateTransitions += total - pending; + if (summary.totalFrames !== null && total > 0) { + summary.framesRendered += Math.floor((summary.totalFrames * succeeded) / total); + } + if (failed > 0) summary.errors.push(...(await failedChunkErrors(sfn, mapRunArn, memoryMb))); +} + +async function failedChunkErrors( + sfn: SFNClient, + mapRunArn: string, + memoryMb: number, +): Promise { + const failed = await sfn.send( + new ListExecutionsCommand({ + mapRunArn, + statusFilter: "FAILED", + maxResults: FAILED_CHUNK_HISTORIES_READ, + }), + ); + const errors: RenderError[] = []; + for (const child of failed.executions ?? []) { + if (!child.executionArn) continue; + const childHistory = await loadFullHistory(sfn, child.executionArn); + errors.push(...summarizeHistory(childHistory, memoryMb).errors); + } + return errors; } interface HistorySummary { @@ -396,6 +449,10 @@ function isRenderChunkState(name: string | null | undefined): boolean { return name === "RenderChunk" || name === "RenderChunkV2"; } +function isRenderChunksMapState(name: string | null | undefined): boolean { + return name === "RenderChunks" || name === "RenderChunksV2"; +} + function isAssembleState(name: string | null | undefined): boolean { return name === "Assemble" || name === "AssembleV2"; } diff --git a/packages/cli/src/commands/lambda/policies.ts b/packages/cli/src/commands/lambda/policies.ts index e1f12d0790..72a3567e3d 100644 --- a/packages/cli/src/commands/lambda/policies.ts +++ b/packages/cli/src/commands/lambda/policies.ts @@ -144,6 +144,7 @@ const REQUIRED_ACTIONS = { "states:CreateStateMachine", "states:DeleteStateMachine", "states:DescribeExecution", + "states:DescribeMapRun", "states:DescribeStateMachine", "states:GetExecutionHistory", "states:ListExecutions",