diff --git a/.github/workflows/lambda-sam-template.yml b/.github/workflows/lambda-sam-template.yml new file mode 100644 index 0000000000..6e217f6b8c --- /dev/null +++ b/.github/workflows/lambda-sam-template.yml @@ -0,0 +1,25 @@ +name: Lambda SAM template + +on: + pull_request: + paths: + - "examples/aws-lambda/**" + - ".github/workflows/lambda-sam-template.yml" + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + sam-deploy: + name: sam deploy validation + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + - run: pipx install aws-sam-cli==1.169.0 + - run: bash examples/aws-lambda/scripts/check-sam-deploy.sh diff --git a/examples/aws-lambda/scripts/check-sam-deploy.sh b/examples/aws-lambda/scripts/check-sam-deploy.sh new file mode 100644 index 0000000000..1a4d3bead7 --- /dev/null +++ b/examples/aws-lambda/scripts/check-sam-deploy.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# `sam deploy` validates the template before it calls AWS. With fake credentials it must get past that +# validation and fail at the first upload, with and without a video image (an image function at the +# top level of template.yaml, for one, fails validation for every deploy). +set -uo pipefail +cd "$(dirname "$0")/.." +work=$(mktemp -d) +trap 'rm -rf "$work"' EXIT +mkdir -p "$work/examples/aws-lambda" "$work/packages/aws-lambda/dist" +cp template.yaml video-function.yaml "$work/examples/aws-lambda/" +echo zip > "$work/packages/aws-lambda/dist/handler.zip" +cd "$work/examples/aws-lambda" +sam validate --lint --region us-east-1 -t template.yaml || exit 1 +sam validate --lint --region us-east-1 -t video-function.yaml || exit 1 +digest=sha256:$(printf '%064d' 0) +for uri in '""' "123456789012.dkr.ecr.us-east-1.amazonaws.com/hf-video@$digest"; do + # The CLI's argv (sam.ts), with --s3-bucket in place of --resolve-s3 so nothing calls AWS before the upload. + out=$(env -u AWS_PROFILE AWS_ACCESS_KEY_ID=AKIAFAKEFAKEFAKEFAKE AWS_SECRET_ACCESS_KEY=fake \ + sam deploy --stack-name hf-sam-check --region us-east-1 --s3-bucket hf-sam-check-none \ + --capabilities CAPABILITY_IAM --no-confirm-changeset --no-fail-on-empty-changeset \ + --parameter-overrides ChromeSource=sparticuz "VideoImageUri=$uri" &1) + if ! grep -q "Unable to upload artifact" <<<"$out"; then + echo "sam deploy stopped before reaching AWS (VideoImageUri=$uri):" + echo "$out" | tail -5 + exit 1 + fi + echo "VideoImageUri=$uri: past validation, stopped at the upload" +done diff --git a/examples/aws-lambda/template.yaml b/examples/aws-lambda/template.yaml index 807bbdeccf..4a03fe96f5 100644 --- a/examples/aws-lambda/template.yaml +++ b/examples/aws-lambda/template.yaml @@ -85,9 +85,8 @@ Globals: # terminate at the Step Functions → Lambda boundary instead of # following into per-function spans. Tracing: Active - # Cost-allocation tags for every AWS::Serverless::Function (the render - # function and the optional video function). Bucket + state-machine carry - # the same tags resource-locally because Globals only covers functions. + # Cost-allocation tags for the render function. Bucket, state machine and + # video-function.yaml carry the same tags themselves; Globals covers only SAM functions. Tags: Project: !Ref ProjectName HyperFramesComponent: lambda-renderer @@ -151,7 +150,7 @@ Resources: HyperFrames distributed render handler. Dispatches on event.Action. Handler: handler.handler Runtime: nodejs22.x - # The ZIP's @sparticuz/chromium ships x86_64 Chrome only; arm64 runs as VideoFunction's image. + # The ZIP's @sparticuz/chromium ships x86_64 Chrome only; video-function.yaml runs chunks on arm64. Architectures: [x86_64] # Local path is resolved by `sam build` + `sam deploy --resolve-s3` # (or `--s3-bucket`); the resulting CodeUri rewrites to s3://. @@ -190,34 +189,25 @@ Resources: # IAM into adopters' accounts. # ── Optional arm64 chunk renderer: the same handler as a container image ── - VideoFunction: - Type: AWS::Serverless::Function + # Nested: `sam deploy` demands an image repository for any image function in this template. + VideoFunctionStack: + Type: AWS::CloudFormation::Stack Condition: HasVideoFunction Properties: - FunctionName: !Sub "${ProjectName}-video" - Description: >- - HyperFrames distributed render handler on arm64, invoked for renderChunk only. - PackageType: Image - ImageUri: !Ref VideoImageUri - Architectures: [arm64] - ReservedConcurrentExecutions: !If - - HasReservedConcurrency - - !Ref ReservedConcurrency - - !Ref AWS::NoValue - EphemeralStorage: - Size: 10240 - Environment: - Variables: - TMPDIR: /tmp - HYPERFRAMES_RENDER_BUCKET: !Ref RenderBucket - HYPERFRAMES_LAMBDA_RESERVED_CONCURRENCY: !Ref ReservedConcurrency - Policies: - - S3CrudPolicy: - BucketName: !Ref RenderBucket - - Statement: - - Effect: Allow - Action: s3:AbortMultipartUpload - Resource: !Sub "arn:${AWS::Partition}:s3:::${RenderBucket}/*" + TemplateURL: video-function.yaml + Parameters: + ProjectName: !Ref ProjectName + ImageUri: !Ref VideoImageUri + RenderBucket: !Ref RenderBucket + MemorySize: !Ref LambdaMemoryMb + Timeout: !Ref LambdaTimeoutSec + ReservedConcurrency: !Ref ReservedConcurrency + ChromeSource: !Ref ChromeSource + Tags: + - Key: Project + Value: !Ref ProjectName + - Key: HyperFramesComponent + Value: lambda-renderer # ── CloudWatch log group for the state machine ────────────────────────── # SAM doesn't auto-create one when `LoggingConfiguration` is set, so we @@ -460,7 +450,11 @@ Resources: Resource: arn:aws:states:::lambda:invoke Parameters: FunctionName: - !If [HasVideoFunction, !GetAtt VideoFunction.Arn, !GetAtt RenderFunction.Arn] + !If [ + HasVideoFunction, + !GetAtt VideoFunctionStack.Outputs.FunctionArn, + !GetAtt RenderFunction.Arn, + ] Payload: Action: renderChunk PlanProtocol: v1 @@ -581,7 +575,7 @@ Resources: FunctionName: !If [ HasVideoFunction, - !GetAtt VideoFunction.Arn, + !GetAtt VideoFunctionStack.Outputs.FunctionArn, !GetAtt RenderFunction.Arn, ] Payload: @@ -768,7 +762,11 @@ Resources: Action: lambda:InvokeFunction Resource: - !GetAtt RenderFunction.Arn - - !If [HasVideoFunction, !GetAtt VideoFunction.Arn, !Ref AWS::NoValue] + - !If [ + HasVideoFunction, + !GetAtt VideoFunctionStack.Outputs.FunctionArn, + !Ref AWS::NoValue, + ] # A Distributed Map runs each chunk as a child execution of this state machine. # ARNs built by name: !Ref RenderStateMachine here would be a dependency cycle. - PolicyName: RunDistributedMapChildren @@ -823,7 +821,12 @@ Resources: MetricName: Invocations Dimensions: - Name: FunctionName - Value: !If [HasVideoFunction, !Ref VideoFunction, !Ref RenderFunction] + Value: + !If [ + HasVideoFunction, + !GetAtt VideoFunctionStack.Outputs.FunctionName, + !Ref RenderFunction, + ] Statistic: Sum Period: 3600 EvaluationPeriods: 1 @@ -867,7 +870,7 @@ Resources: MetricName: Errors Dimensions: - Name: FunctionName - Value: !Ref VideoFunction + Value: !GetAtt VideoFunctionStack.Outputs.FunctionName Statistic: Sum Period: 300 EvaluationPeriods: 1 @@ -918,7 +921,7 @@ Outputs: VideoFunctionArn: Condition: HasVideoFunction Description: ARN of the arm64 chunk-rendering function. - Value: !GetAtt VideoFunction.Arn + Value: !GetAtt VideoFunctionStack.Outputs.FunctionArn RenderStateMachineArn: Description: ARN of the Step Functions state machine. Pass to `aws stepfunctions start-execution`. diff --git a/examples/aws-lambda/video-function.yaml b/examples/aws-lambda/video-function.yaml new file mode 100644 index 0000000000..98fd7198e5 --- /dev/null +++ b/examples/aws-lambda/video-function.yaml @@ -0,0 +1,104 @@ +AWSTemplateFormatVersion: "2010-09-09" +Description: >- + HyperFrames arm64 chunk renderer: the render handler as a container image, invoked for renderChunk + only. Nested in template.yaml when VideoImageUri is set; plain CloudFormation, because `sam deploy` + demands an image repository for any top-level image function, even one whose Condition is false. + +Parameters: + ProjectName: + Type: String + ImageUri: + Type: String + RenderBucket: + Type: String + MemorySize: + Type: Number + Timeout: + Type: Number + ReservedConcurrency: + Type: Number + ChromeSource: + Type: String + +Conditions: + HasReservedConcurrency: !Not [!Equals [!Ref ReservedConcurrency, -1]] + +Resources: + Role: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Principal: + Service: lambda.amazonaws.com + Action: sts:AssumeRole + ManagedPolicyArns: + - !Sub "arn:${AWS::Partition}:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" + - !Sub "arn:${AWS::Partition}:iam::aws:policy/AWSXRayDaemonWriteAccess" + # The render function's S3CrudPolicy plus AbortMultipartUpload, spelled out. + Policies: + - PolicyName: render-bucket + PolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Action: + - s3:GetObject + - s3:ListBucket + - s3:GetBucketLocation + - s3:GetObjectVersion + - s3:PutObject + - s3:PutObjectAcl + - s3:GetLifecycleConfiguration + - s3:PutLifecycleConfiguration + - s3:DeleteObject + - s3:AbortMultipartUpload + Resource: + - !Sub "arn:${AWS::Partition}:s3:::${RenderBucket}" + - !Sub "arn:${AWS::Partition}:s3:::${RenderBucket}/*" + Tags: + - Key: Project + Value: !Ref ProjectName + - Key: HyperFramesComponent + Value: lambda-renderer + + Function: + Type: AWS::Lambda::Function + Properties: + FunctionName: !Sub "${ProjectName}-video" + Description: HyperFrames distributed render handler on arm64, invoked for renderChunk only. + PackageType: Image + Code: + ImageUri: !Ref ImageUri + Architectures: [arm64] + Role: !GetAtt Role.Arn + MemorySize: !Ref MemorySize + Timeout: !Ref Timeout + ReservedConcurrentExecutions: !If + - HasReservedConcurrency + - !Ref ReservedConcurrency + - !Ref AWS::NoValue + EphemeralStorage: + Size: 10240 + TracingConfig: + Mode: Active + Environment: + Variables: + NODE_OPTIONS: "--enable-source-maps" + HYPERFRAMES_LAMBDA_CHROME_SOURCE: !Ref ChromeSource + TMPDIR: /tmp + HYPERFRAMES_RENDER_BUCKET: !Ref RenderBucket + HYPERFRAMES_LAMBDA_RESERVED_CONCURRENCY: !Ref ReservedConcurrency + Tags: + - Key: Project + Value: !Ref ProjectName + - Key: HyperFramesComponent + Value: lambda-renderer + +Outputs: + FunctionArn: + Value: !GetAtt Function.Arn + FunctionName: + Value: !Ref Function diff --git a/packages/aws-lambda/scripts/pin-handler-zip.test.ts b/packages/aws-lambda/scripts/pin-handler-zip.test.ts index 15d16dd1e8..a57573d47e 100644 --- a/packages/aws-lambda/scripts/pin-handler-zip.test.ts +++ b/packages/aws-lambda/scripts/pin-handler-zip.test.ts @@ -6,10 +6,9 @@ import { join } from "node:path"; import { adapterPackageRoot, prebuiltHandler } from "../src/sdk/prebuiltHandler.js"; import { pinHandlerZip } from "./pin-handler-zip.js"; -const realTemplate = readFileSync( - join(import.meta.dir, "..", "..", "..", "examples", "aws-lambda", "template.yaml"), - "utf-8", -); +const examples = join(import.meta.dir, "..", "..", "..", "examples", "aws-lambda"); +const realTemplate = readFileSync(join(examples, "template.yaml"), "utf-8"); +const videoTemplate = readFileSync(join(examples, "video-function.yaml"), "utf-8"); const dirs: string[] = []; function fixtureRepo(): { root: string; pkgDir: string } { @@ -27,6 +26,7 @@ function fixtureRepo(): { root: string; pkgDir: string } { ); mkdirSync(join(root, "examples", "aws-lambda"), { recursive: true }); writeFileSync(join(root, "examples", "aws-lambda", "template.yaml"), realTemplate); + writeFileSync(join(root, "examples", "aws-lambda", "video-function.yaml"), videoTemplate); return { root, pkgDir }; } @@ -66,6 +66,10 @@ describe("pinHandlerZip", () => { const template = readFileSync(pinned!.templatePath, "utf-8"); expect(template).toContain(" CodeUri: handler.zip\n"); expect(template).not.toContain("packages/aws-lambda/dist/handler.zip\n"); + expect(template).toContain(" TemplateURL: video-function.yaml\n"); + expect(readFileSync(join(pkgDir, "prebuilt", "video-function.yaml"), "utf-8")).toBe( + videoTemplate, + ); }); it("pins a zip a previous run already attached to the release", () => { diff --git a/packages/aws-lambda/scripts/prebuilt-template.mjs b/packages/aws-lambda/scripts/prebuilt-template.mjs index f2f063dc6e..0f37dc9adb 100644 --- a/packages/aws-lambda/scripts/prebuilt-template.mjs +++ b/packages/aws-lambda/scripts/prebuilt-template.mjs @@ -1,11 +1,15 @@ -import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { copyFileSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; import { join } from "node:path"; const CHECKOUT_CODE_URI = "CodeUri: ../../packages/aws-lambda/dist/handler.zip"; -/** Writes `/template.yaml`: the example SAM template, pointed at a `handler.zip` beside it. */ +/** + * Writes `/template.yaml`, the example SAM template pointed at a `handler.zip` beside it, and + * the `video-function.yaml` it nests. + */ export function writePrebuiltTemplate(repoRoot, outDir) { - const template = readFileSync(join(repoRoot, "examples", "aws-lambda", "template.yaml"), "utf-8"); + const examples = join(repoRoot, "examples", "aws-lambda"); + const template = readFileSync(join(examples, "template.yaml"), "utf-8"); if (template.split(CHECKOUT_CODE_URI).length !== 2) { throw new Error( `[prebuilt-template] expected exactly one "${CHECKOUT_CODE_URI}" in the template`, @@ -16,4 +20,5 @@ export function writePrebuiltTemplate(repoRoot, outDir) { join(outDir, "template.yaml"), template.replace(CHECKOUT_CODE_URI, "CodeUri: handler.zip"), ); + copyFileSync(join(examples, "video-function.yaml"), join(outDir, "video-function.yaml")); } diff --git a/packages/aws-lambda/src/cdk/HyperframesRenderStack.snapshot.test.ts b/packages/aws-lambda/src/cdk/HyperframesRenderStack.snapshot.test.ts index bc9c5f205f..0b33748b9d 100644 --- a/packages/aws-lambda/src/cdk/HyperframesRenderStack.snapshot.test.ts +++ b/packages/aws-lambda/src/cdk/HyperframesRenderStack.snapshot.test.ts @@ -360,7 +360,7 @@ describe("HyperframesRenderStack — snapshot", () => { action, functionName: action === "renderChunk" - ? ["HasVideoFunction", "VideoFunction.Arn", "RenderFunction.Arn"] + ? ["HasVideoFunction", "VideoFunctionStack.Outputs.FunctionArn", "RenderFunction.Arn"] : "RenderFunction.Arn", }); } @@ -368,7 +368,21 @@ describe("HyperframesRenderStack — snapshot", () => { const invoke = JSON.stringify( requireRecordProperty(samRole, "Properties", "SAM role properties").Policies, ); - expect(invoke).toContain('["RenderFunction.Arn",["HasVideoFunction","VideoFunction.Arn"'); + expect(invoke).toContain( + '["RenderFunction.Arn",["HasVideoFunction","VideoFunctionStack.Outputs.FunctionArn"', + ); + }); + + it("gives the nested arm64 video function every environment variable the render function gets", () => { + const sam = readExampleTemplate("template.yaml"); + const render = [ + ...envNames(sam, "Globals", "Function", "Environment", "Variables"), + ...envNames(sam, "Resources", "RenderFunction", "Properties", "Environment", "Variables"), + ]; + const video = readExampleTemplate("video-function.yaml"); + expect( + envNames(video, "Resources", "Function", "Properties", "Environment", "Variables").sort(), + ).toEqual([...new Set(render)].sort()); }); it("lets the render function abort the multipart uploads its S3 stitch starts", () => { @@ -689,17 +703,28 @@ function policyStatements(policy: unknown): Record[] { return document.Statement as Record[]; } -function readSamResources(): Record { +function readExampleTemplate(file: string): Record { const source = readFileSync( - new URL("../../../../examples/aws-lambda/template.yaml", import.meta.url), + new URL(`../../../../examples/aws-lambda/${file}`, import.meta.url), "utf8", ); // CloudFormation intrinsic tags are irrelevant to classifier parity. The // YAML parser preserves their scalar values while this option suppresses // warnings for the intentionally unresolved `!Ref`/`!GetAtt` tags. const parsed: unknown = parseYaml(source, { logLevel: "silent" }); - const root = requireRecord(parsed, "SAM template"); - return requireRecordProperty(root, "Resources", "SAM resources"); + return requireRecord(parsed, file); +} + +function readSamResources(): Record { + return requireRecordProperty(readExampleTemplate("template.yaml"), "Resources", "SAM resources"); +} + +function envNames(root: Record, ...path: string[]): string[] { + const variables = path.reduce>( + (node, key) => requireRecordProperty(node, key, key), + root, + ); + return Object.keys(variables); } function readSamDefinition(): { States: Record } { diff --git a/packages/cli/src/commands/lambda/prebuiltHandler.test.ts b/packages/cli/src/commands/lambda/prebuiltHandler.test.ts index 9bf966a8d2..9817463e6c 100644 --- a/packages/cli/src/commands/lambda/prebuiltHandler.test.ts +++ b/packages/cli/src/commands/lambda/prebuiltHandler.test.ts @@ -15,6 +15,7 @@ function setup(body: Buffer = ZIP) { dirs.push(root); const templatePath = join(root, "template.yaml"); writeFileSync(templatePath, "CodeUri: handler.zip\n"); + writeFileSync(join(root, "video-function.yaml"), "PackageType: Image\n"); const downloads: string[] = []; const logs: string[] = []; const deps: StageDeps = { @@ -57,6 +58,7 @@ describe("stagePrebuiltHandler", () => { expect(dir).toBe(join(deps.cacheDir, `9.8.7-${SHA}`)); expect(readFileSync(join(dir, "handler.zip"))).toEqual(ZIP); expect(readFileSync(join(dir, "template.yaml"), "utf-8")).toBe("CodeUri: handler.zip\n"); + expect(readFileSync(join(dir, "video-function.yaml"), "utf-8")).toBe("PackageType: Image\n"); expect(await stagePrebuiltHandler(deps)).toBe(dir); expect(downloads).toEqual([URL]); diff --git a/packages/cli/src/commands/lambda/prebuiltHandler.ts b/packages/cli/src/commands/lambda/prebuiltHandler.ts index 5849ba1f68..a1b4e178ec 100644 --- a/packages/cli/src/commands/lambda/prebuiltHandler.ts +++ b/packages/cli/src/commands/lambda/prebuiltHandler.ts @@ -2,9 +2,17 @@ // stage it beside the packaged SAM template. import { createHash } from "node:crypto"; -import { copyFileSync, existsSync, mkdirSync, readFileSync, renameSync, unlinkSync } from "node:fs"; +import { + copyFileSync, + existsSync, + mkdirSync, + readdirSync, + readFileSync, + renameSync, + unlinkSync, +} from "node:fs"; import { homedir } from "node:os"; -import { join } from "node:path"; +import { dirname, join } from "node:path"; import { downloadFile } from "../../utils/download.js"; import { isMissingPackageError } from "../../utils/missingPackage.js"; import { VERSION } from "../../version.js"; @@ -83,6 +91,10 @@ export async function stagePrebuiltHandler(deps: StageDeps = defaultDeps): Promi renameSync(partial, zip); deps.log(`→ Verified sha256 ${pinned.sha256}`); } - copyFileSync(pinned.templatePath, join(dir, "template.yaml")); + // The template and the nested templates it deploys (video-function.yaml). + const templates = dirname(pinned.templatePath); + for (const file of readdirSync(templates).filter((name) => name.endsWith(".yaml"))) { + copyFileSync(join(templates, file), join(dir, file)); + } return dir; }