From f9bfde86d3a3ed3253ae7117b5cb6185d72cf643 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81ngel?= Date: Thu, 22 Oct 2026 12:58:47 -0400 Subject: [PATCH] fix(aws-lambda): nest the arm64 video function so sam deploy needs no image repository sam deploy refuses a template with any top-level image function unless it is given an image repository, even when that function's Condition is false, so every deploy of the template failed. The video function now lives in video-function.yaml, a nested stack created only when VideoImageUri is set. The release ships it beside template.yaml and the CLI stages both. A new workflow runs the CLI's sam deploy argv against the template with fake credentials and requires it to reach the first upload. --- .github/workflows/lambda-sam-template.yml | 25 +++++ .../aws-lambda/scripts/check-sam-deploy.sh | 28 +++++ examples/aws-lambda/template.yaml | 75 +++++++------ examples/aws-lambda/video-function.yaml | 104 ++++++++++++++++++ .../scripts/pin-handler-zip.test.ts | 12 +- .../aws-lambda/scripts/prebuilt-template.mjs | 11 +- .../HyperframesRenderStack.snapshot.test.ts | 37 ++++++- .../commands/lambda/prebuiltHandler.test.ts | 2 + .../src/commands/lambda/prebuiltHandler.ts | 18 ++- 9 files changed, 260 insertions(+), 52 deletions(-) create mode 100644 .github/workflows/lambda-sam-template.yml create mode 100644 examples/aws-lambda/scripts/check-sam-deploy.sh create mode 100644 examples/aws-lambda/video-function.yaml diff --git a/.github/workflows/lambda-sam-template.yml b/.github/workflows/lambda-sam-template.yml new file mode 100644 index 0000000000..6e217f6b8c --- /dev/null +++ b/.github/workflows/lambda-sam-template.yml @@ -0,0 +1,25 @@ +name: Lambda SAM template + +on: + pull_request: + paths: + - "examples/aws-lambda/**" + - ".github/workflows/lambda-sam-template.yml" + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + sam-deploy: + name: sam deploy validation + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + - run: pipx install aws-sam-cli==1.169.0 + - run: bash examples/aws-lambda/scripts/check-sam-deploy.sh diff --git a/examples/aws-lambda/scripts/check-sam-deploy.sh b/examples/aws-lambda/scripts/check-sam-deploy.sh new file mode 100644 index 0000000000..1a4d3bead7 --- /dev/null +++ b/examples/aws-lambda/scripts/check-sam-deploy.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# `sam deploy` validates the template before it calls AWS. With fake credentials it must get past that +# validation and fail at the first upload, with and without a video image (an image function at the +# top level of template.yaml, for one, fails validation for every deploy). +set -uo pipefail +cd "$(dirname "$0")/.." +work=$(mktemp -d) +trap 'rm -rf "$work"' EXIT +mkdir -p "$work/examples/aws-lambda" "$work/packages/aws-lambda/dist" +cp template.yaml video-function.yaml "$work/examples/aws-lambda/" +echo zip > "$work/packages/aws-lambda/dist/handler.zip" +cd "$work/examples/aws-lambda" +sam validate --lint --region us-east-1 -t template.yaml || exit 1 +sam validate --lint --region us-east-1 -t video-function.yaml || exit 1 +digest=sha256:$(printf '%064d' 0) +for uri in '""' "123456789012.dkr.ecr.us-east-1.amazonaws.com/hf-video@$digest"; do + # The CLI's argv (sam.ts), with --s3-bucket in place of --resolve-s3 so nothing calls AWS before the upload. + out=$(env -u AWS_PROFILE AWS_ACCESS_KEY_ID=AKIAFAKEFAKEFAKEFAKE AWS_SECRET_ACCESS_KEY=fake \ + sam deploy --stack-name hf-sam-check --region us-east-1 --s3-bucket hf-sam-check-none \ + --capabilities CAPABILITY_IAM --no-confirm-changeset --no-fail-on-empty-changeset \ + --parameter-overrides ChromeSource=sparticuz "VideoImageUri=$uri" &1) + if ! grep -q "Unable to upload artifact" <<<"$out"; then + echo "sam deploy stopped before reaching AWS (VideoImageUri=$uri):" + echo "$out" | tail -5 + exit 1 + fi + echo "VideoImageUri=$uri: past validation, stopped at the upload" +done diff --git a/examples/aws-lambda/template.yaml b/examples/aws-lambda/template.yaml index 807bbdeccf..4a03fe96f5 100644 --- a/examples/aws-lambda/template.yaml +++ b/examples/aws-lambda/template.yaml @@ -85,9 +85,8 @@ Globals: # terminate at the Step Functions → Lambda boundary instead of # following into per-function spans. Tracing: Active - # Cost-allocation tags for every AWS::Serverless::Function (the render - # function and the optional video function). Bucket + state-machine carry - # the same tags resource-locally because Globals only covers functions. + # Cost-allocation tags for the render function. Bucket, state machine and + # video-function.yaml carry the same tags themselves; Globals covers only SAM functions. Tags: Project: !Ref ProjectName HyperFramesComponent: lambda-renderer @@ -151,7 +150,7 @@ Resources: HyperFrames distributed render handler. Dispatches on event.Action. Handler: handler.handler Runtime: nodejs22.x - # The ZIP's @sparticuz/chromium ships x86_64 Chrome only; arm64 runs as VideoFunction's image. + # The ZIP's @sparticuz/chromium ships x86_64 Chrome only; video-function.yaml runs chunks on arm64. Architectures: [x86_64] # Local path is resolved by `sam build` + `sam deploy --resolve-s3` # (or `--s3-bucket`); the resulting CodeUri rewrites to s3://. @@ -190,34 +189,25 @@ Resources: # IAM into adopters' accounts. # ── Optional arm64 chunk renderer: the same handler as a container image ── - VideoFunction: - Type: AWS::Serverless::Function + # Nested: `sam deploy` demands an image repository for any image function in this template. + VideoFunctionStack: + Type: AWS::CloudFormation::Stack Condition: HasVideoFunction Properties: - FunctionName: !Sub "${ProjectName}-video" - Description: >- - HyperFrames distributed render handler on arm64, invoked for renderChunk only. - PackageType: Image - ImageUri: !Ref VideoImageUri - Architectures: [arm64] - ReservedConcurrentExecutions: !If - - HasReservedConcurrency - - !Ref ReservedConcurrency - - !Ref AWS::NoValue - EphemeralStorage: - Size: 10240 - Environment: - Variables: - TMPDIR: /tmp - HYPERFRAMES_RENDER_BUCKET: !Ref RenderBucket - HYPERFRAMES_LAMBDA_RESERVED_CONCURRENCY: !Ref ReservedConcurrency - Policies: - - S3CrudPolicy: - BucketName: !Ref RenderBucket - - Statement: - - Effect: Allow - Action: s3:AbortMultipartUpload - Resource: !Sub "arn:${AWS::Partition}:s3:::${RenderBucket}/*" + TemplateURL: video-function.yaml + Parameters: + ProjectName: !Ref ProjectName + ImageUri: !Ref VideoImageUri + RenderBucket: !Ref RenderBucket + MemorySize: !Ref LambdaMemoryMb + Timeout: !Ref LambdaTimeoutSec + ReservedConcurrency: !Ref ReservedConcurrency + ChromeSource: !Ref ChromeSource + Tags: + - Key: Project + Value: !Ref ProjectName + - Key: HyperFramesComponent + Value: lambda-renderer # ── CloudWatch log group for the state machine ────────────────────────── # SAM doesn't auto-create one when `LoggingConfiguration` is set, so we @@ -460,7 +450,11 @@ Resources: Resource: arn:aws:states:::lambda:invoke Parameters: FunctionName: - !If [HasVideoFunction, !GetAtt VideoFunction.Arn, !GetAtt RenderFunction.Arn] + !If [ + HasVideoFunction, + !GetAtt VideoFunctionStack.Outputs.FunctionArn, + !GetAtt RenderFunction.Arn, + ] Payload: Action: renderChunk PlanProtocol: v1 @@ -581,7 +575,7 @@ Resources: FunctionName: !If [ HasVideoFunction, - !GetAtt VideoFunction.Arn, + !GetAtt VideoFunctionStack.Outputs.FunctionArn, !GetAtt RenderFunction.Arn, ] Payload: @@ -768,7 +762,11 @@ Resources: Action: lambda:InvokeFunction Resource: - !GetAtt RenderFunction.Arn - - !If [HasVideoFunction, !GetAtt VideoFunction.Arn, !Ref AWS::NoValue] + - !If [ + HasVideoFunction, + !GetAtt VideoFunctionStack.Outputs.FunctionArn, + !Ref AWS::NoValue, + ] # A Distributed Map runs each chunk as a child execution of this state machine. # ARNs built by name: !Ref RenderStateMachine here would be a dependency cycle. - PolicyName: RunDistributedMapChildren @@ -823,7 +821,12 @@ Resources: MetricName: Invocations Dimensions: - Name: FunctionName - Value: !If [HasVideoFunction, !Ref VideoFunction, !Ref RenderFunction] + Value: + !If [ + HasVideoFunction, + !GetAtt VideoFunctionStack.Outputs.FunctionName, + !Ref RenderFunction, + ] Statistic: Sum Period: 3600 EvaluationPeriods: 1 @@ -867,7 +870,7 @@ Resources: MetricName: Errors Dimensions: - Name: FunctionName - Value: !Ref VideoFunction + Value: !GetAtt VideoFunctionStack.Outputs.FunctionName Statistic: Sum Period: 300 EvaluationPeriods: 1 @@ -918,7 +921,7 @@ Outputs: VideoFunctionArn: Condition: HasVideoFunction Description: ARN of the arm64 chunk-rendering function. - Value: !GetAtt VideoFunction.Arn + Value: !GetAtt VideoFunctionStack.Outputs.FunctionArn RenderStateMachineArn: Description: ARN of the Step Functions state machine. Pass to `aws stepfunctions start-execution`. diff --git a/examples/aws-lambda/video-function.yaml b/examples/aws-lambda/video-function.yaml new file mode 100644 index 0000000000..98fd7198e5 --- /dev/null +++ b/examples/aws-lambda/video-function.yaml @@ -0,0 +1,104 @@ +AWSTemplateFormatVersion: "2010-09-09" +Description: >- + HyperFrames arm64 chunk renderer: the render handler as a container image, invoked for renderChunk + only. Nested in template.yaml when VideoImageUri is set; plain CloudFormation, because `sam deploy` + demands an image repository for any top-level image function, even one whose Condition is false. + +Parameters: + ProjectName: + Type: String + ImageUri: + Type: String + RenderBucket: + Type: String + MemorySize: + Type: Number + Timeout: + Type: Number + ReservedConcurrency: + Type: Number + ChromeSource: + Type: String + +Conditions: + HasReservedConcurrency: !Not [!Equals [!Ref ReservedConcurrency, -1]] + +Resources: + Role: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Principal: + Service: lambda.amazonaws.com + Action: sts:AssumeRole + ManagedPolicyArns: + - !Sub "arn:${AWS::Partition}:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" + - !Sub "arn:${AWS::Partition}:iam::aws:policy/AWSXRayDaemonWriteAccess" + # The render function's S3CrudPolicy plus AbortMultipartUpload, spelled out. + Policies: + - PolicyName: render-bucket + PolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Action: + - s3:GetObject + - s3:ListBucket + - s3:GetBucketLocation + - s3:GetObjectVersion + - s3:PutObject + - s3:PutObjectAcl + - s3:GetLifecycleConfiguration + - s3:PutLifecycleConfiguration + - s3:DeleteObject + - s3:AbortMultipartUpload + Resource: + - !Sub "arn:${AWS::Partition}:s3:::${RenderBucket}" + - !Sub "arn:${AWS::Partition}:s3:::${RenderBucket}/*" + Tags: + - Key: Project + Value: !Ref ProjectName + - Key: HyperFramesComponent + Value: lambda-renderer + + Function: + Type: AWS::Lambda::Function + Properties: + FunctionName: !Sub "${ProjectName}-video" + Description: HyperFrames distributed render handler on arm64, invoked for renderChunk only. + PackageType: Image + Code: + ImageUri: !Ref ImageUri + Architectures: [arm64] + Role: !GetAtt Role.Arn + MemorySize: !Ref MemorySize + Timeout: !Ref Timeout + ReservedConcurrentExecutions: !If + - HasReservedConcurrency + - !Ref ReservedConcurrency + - !Ref AWS::NoValue + EphemeralStorage: + Size: 10240 + TracingConfig: + Mode: Active + Environment: + Variables: + NODE_OPTIONS: "--enable-source-maps" + HYPERFRAMES_LAMBDA_CHROME_SOURCE: !Ref ChromeSource + TMPDIR: /tmp + HYPERFRAMES_RENDER_BUCKET: !Ref RenderBucket + HYPERFRAMES_LAMBDA_RESERVED_CONCURRENCY: !Ref ReservedConcurrency + Tags: + - Key: Project + Value: !Ref ProjectName + - Key: HyperFramesComponent + Value: lambda-renderer + +Outputs: + FunctionArn: + Value: !GetAtt Function.Arn + FunctionName: + Value: !Ref Function diff --git a/packages/aws-lambda/scripts/pin-handler-zip.test.ts b/packages/aws-lambda/scripts/pin-handler-zip.test.ts index 15d16dd1e8..a57573d47e 100644 --- a/packages/aws-lambda/scripts/pin-handler-zip.test.ts +++ b/packages/aws-lambda/scripts/pin-handler-zip.test.ts @@ -6,10 +6,9 @@ import { join } from "node:path"; import { adapterPackageRoot, prebuiltHandler } from "../src/sdk/prebuiltHandler.js"; import { pinHandlerZip } from "./pin-handler-zip.js"; -const realTemplate = readFileSync( - join(import.meta.dir, "..", "..", "..", "examples", "aws-lambda", "template.yaml"), - "utf-8", -); +const examples = join(import.meta.dir, "..", "..", "..", "examples", "aws-lambda"); +const realTemplate = readFileSync(join(examples, "template.yaml"), "utf-8"); +const videoTemplate = readFileSync(join(examples, "video-function.yaml"), "utf-8"); const dirs: string[] = []; function fixtureRepo(): { root: string; pkgDir: string } { @@ -27,6 +26,7 @@ function fixtureRepo(): { root: string; pkgDir: string } { ); mkdirSync(join(root, "examples", "aws-lambda"), { recursive: true }); writeFileSync(join(root, "examples", "aws-lambda", "template.yaml"), realTemplate); + writeFileSync(join(root, "examples", "aws-lambda", "video-function.yaml"), videoTemplate); return { root, pkgDir }; } @@ -66,6 +66,10 @@ describe("pinHandlerZip", () => { const template = readFileSync(pinned!.templatePath, "utf-8"); expect(template).toContain(" CodeUri: handler.zip\n"); expect(template).not.toContain("packages/aws-lambda/dist/handler.zip\n"); + expect(template).toContain(" TemplateURL: video-function.yaml\n"); + expect(readFileSync(join(pkgDir, "prebuilt", "video-function.yaml"), "utf-8")).toBe( + videoTemplate, + ); }); it("pins a zip a previous run already attached to the release", () => { diff --git a/packages/aws-lambda/scripts/prebuilt-template.mjs b/packages/aws-lambda/scripts/prebuilt-template.mjs index f2f063dc6e..0f37dc9adb 100644 --- a/packages/aws-lambda/scripts/prebuilt-template.mjs +++ b/packages/aws-lambda/scripts/prebuilt-template.mjs @@ -1,11 +1,15 @@ -import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { copyFileSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; import { join } from "node:path"; const CHECKOUT_CODE_URI = "CodeUri: ../../packages/aws-lambda/dist/handler.zip"; -/** Writes `/template.yaml`: the example SAM template, pointed at a `handler.zip` beside it. */ +/** + * Writes `/template.yaml`, the example SAM template pointed at a `handler.zip` beside it, and + * the `video-function.yaml` it nests. + */ export function writePrebuiltTemplate(repoRoot, outDir) { - const template = readFileSync(join(repoRoot, "examples", "aws-lambda", "template.yaml"), "utf-8"); + const examples = join(repoRoot, "examples", "aws-lambda"); + const template = readFileSync(join(examples, "template.yaml"), "utf-8"); if (template.split(CHECKOUT_CODE_URI).length !== 2) { throw new Error( `[prebuilt-template] expected exactly one "${CHECKOUT_CODE_URI}" in the template`, @@ -16,4 +20,5 @@ export function writePrebuiltTemplate(repoRoot, outDir) { join(outDir, "template.yaml"), template.replace(CHECKOUT_CODE_URI, "CodeUri: handler.zip"), ); + copyFileSync(join(examples, "video-function.yaml"), join(outDir, "video-function.yaml")); } diff --git a/packages/aws-lambda/src/cdk/HyperframesRenderStack.snapshot.test.ts b/packages/aws-lambda/src/cdk/HyperframesRenderStack.snapshot.test.ts index bc9c5f205f..0b33748b9d 100644 --- a/packages/aws-lambda/src/cdk/HyperframesRenderStack.snapshot.test.ts +++ b/packages/aws-lambda/src/cdk/HyperframesRenderStack.snapshot.test.ts @@ -360,7 +360,7 @@ describe("HyperframesRenderStack — snapshot", () => { action, functionName: action === "renderChunk" - ? ["HasVideoFunction", "VideoFunction.Arn", "RenderFunction.Arn"] + ? ["HasVideoFunction", "VideoFunctionStack.Outputs.FunctionArn", "RenderFunction.Arn"] : "RenderFunction.Arn", }); } @@ -368,7 +368,21 @@ describe("HyperframesRenderStack — snapshot", () => { const invoke = JSON.stringify( requireRecordProperty(samRole, "Properties", "SAM role properties").Policies, ); - expect(invoke).toContain('["RenderFunction.Arn",["HasVideoFunction","VideoFunction.Arn"'); + expect(invoke).toContain( + '["RenderFunction.Arn",["HasVideoFunction","VideoFunctionStack.Outputs.FunctionArn"', + ); + }); + + it("gives the nested arm64 video function every environment variable the render function gets", () => { + const sam = readExampleTemplate("template.yaml"); + const render = [ + ...envNames(sam, "Globals", "Function", "Environment", "Variables"), + ...envNames(sam, "Resources", "RenderFunction", "Properties", "Environment", "Variables"), + ]; + const video = readExampleTemplate("video-function.yaml"); + expect( + envNames(video, "Resources", "Function", "Properties", "Environment", "Variables").sort(), + ).toEqual([...new Set(render)].sort()); }); it("lets the render function abort the multipart uploads its S3 stitch starts", () => { @@ -689,17 +703,28 @@ function policyStatements(policy: unknown): Record[] { return document.Statement as Record[]; } -function readSamResources(): Record { +function readExampleTemplate(file: string): Record { const source = readFileSync( - new URL("../../../../examples/aws-lambda/template.yaml", import.meta.url), + new URL(`../../../../examples/aws-lambda/${file}`, import.meta.url), "utf8", ); // CloudFormation intrinsic tags are irrelevant to classifier parity. The // YAML parser preserves their scalar values while this option suppresses // warnings for the intentionally unresolved `!Ref`/`!GetAtt` tags. const parsed: unknown = parseYaml(source, { logLevel: "silent" }); - const root = requireRecord(parsed, "SAM template"); - return requireRecordProperty(root, "Resources", "SAM resources"); + return requireRecord(parsed, file); +} + +function readSamResources(): Record { + return requireRecordProperty(readExampleTemplate("template.yaml"), "Resources", "SAM resources"); +} + +function envNames(root: Record, ...path: string[]): string[] { + const variables = path.reduce>( + (node, key) => requireRecordProperty(node, key, key), + root, + ); + return Object.keys(variables); } function readSamDefinition(): { States: Record } { diff --git a/packages/cli/src/commands/lambda/prebuiltHandler.test.ts b/packages/cli/src/commands/lambda/prebuiltHandler.test.ts index 9bf966a8d2..9817463e6c 100644 --- a/packages/cli/src/commands/lambda/prebuiltHandler.test.ts +++ b/packages/cli/src/commands/lambda/prebuiltHandler.test.ts @@ -15,6 +15,7 @@ function setup(body: Buffer = ZIP) { dirs.push(root); const templatePath = join(root, "template.yaml"); writeFileSync(templatePath, "CodeUri: handler.zip\n"); + writeFileSync(join(root, "video-function.yaml"), "PackageType: Image\n"); const downloads: string[] = []; const logs: string[] = []; const deps: StageDeps = { @@ -57,6 +58,7 @@ describe("stagePrebuiltHandler", () => { expect(dir).toBe(join(deps.cacheDir, `9.8.7-${SHA}`)); expect(readFileSync(join(dir, "handler.zip"))).toEqual(ZIP); expect(readFileSync(join(dir, "template.yaml"), "utf-8")).toBe("CodeUri: handler.zip\n"); + expect(readFileSync(join(dir, "video-function.yaml"), "utf-8")).toBe("PackageType: Image\n"); expect(await stagePrebuiltHandler(deps)).toBe(dir); expect(downloads).toEqual([URL]); diff --git a/packages/cli/src/commands/lambda/prebuiltHandler.ts b/packages/cli/src/commands/lambda/prebuiltHandler.ts index 5849ba1f68..a1b4e178ec 100644 --- a/packages/cli/src/commands/lambda/prebuiltHandler.ts +++ b/packages/cli/src/commands/lambda/prebuiltHandler.ts @@ -2,9 +2,17 @@ // stage it beside the packaged SAM template. import { createHash } from "node:crypto"; -import { copyFileSync, existsSync, mkdirSync, readFileSync, renameSync, unlinkSync } from "node:fs"; +import { + copyFileSync, + existsSync, + mkdirSync, + readdirSync, + readFileSync, + renameSync, + unlinkSync, +} from "node:fs"; import { homedir } from "node:os"; -import { join } from "node:path"; +import { dirname, join } from "node:path"; import { downloadFile } from "../../utils/download.js"; import { isMissingPackageError } from "../../utils/missingPackage.js"; import { VERSION } from "../../version.js"; @@ -83,6 +91,10 @@ export async function stagePrebuiltHandler(deps: StageDeps = defaultDeps): Promi renameSync(partial, zip); deps.log(`→ Verified sha256 ${pinned.sha256}`); } - copyFileSync(pinned.templatePath, join(dir, "template.yaml")); + // The template and the nested templates it deploys (video-function.yaml). + const templates = dirname(pinned.templatePath); + for (const file of readdirSync(templates).filter((name) => name.endsWith(".yaml"))) { + copyFileSync(join(templates, file), join(dir, file)); + } return dir; }