From 61d65e717736eb8918cd99ad1d50be61cbebd986 Mon Sep 17 00:00:00 2001 From: James Russo Date: Wed, 18 Nov 2026 11:25:22 +0000 Subject: [PATCH] fix(aws-lambda): match the shell to its own pin, ship only hashed files, refuse the shell ZIP in CDK and SAM (#8173) * fix(aws-lambda): match the shell to its own pin, ship only hashed files, refuse the shell ZIP in CDK and SAM Review follow-ups to the pinned shell route: - build-zip --source=chrome-headless-shell accepts the folder only when its digest is pinned for this version, not for any pinned build. - It leaves out the files the digest skips (debug.log and chrome_debug.log, which Chrome writes as it runs), so nothing unhashed ships. - The CDK construct throws on chromeSource "chrome-headless-shell" and the SAM template allows only sparticuz, as lambda deploy already refused it: a shell ZIP cannot fit a function. The CHROME_PATH wiring for that ZIP is gone with it. Co-Authored-By: Claude Opus 5.5 * chore(aws-lambda): raise the packed size budget to 82,000 bytes The shipped build-zip script now checks the shell against its own version's pin and skips the files the digest leaves out. That takes the tarball to 81,378 bytes in CI, over the 80,500 budget's 1% tolerance; main was already within about 200 bytes of it. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- examples/aws-lambda/README.md | 2 +- examples/aws-lambda/template.yaml | 14 ++---- packages/aws-lambda/README.md | 10 ++-- .../scripts/_chromeHeadlessShell.ts | 19 +++++--- packages/aws-lambda/scripts/build-zip.test.ts | 46 ++++++++++++------- .../HyperframesRenderStack.snapshot.test.ts | 46 ++++--------------- .../src/cdk/HyperframesRenderStack.ts | 22 ++++----- packages/aws-lambda/src/chromium.ts | 1 - .../engine/src/services/chromeHostCeiling.ts | 2 +- scripts/package-size-budgets.json | 2 +- 10 files changed, 72 insertions(+), 92 deletions(-) diff --git a/examples/aws-lambda/README.md b/examples/aws-lambda/README.md index 51e0f50d0b..210178d4d5 100644 --- a/examples/aws-lambda/README.md +++ b/examples/aws-lambda/README.md @@ -200,7 +200,7 @@ with `--profile ` or `AWS_PROFILE=`. | `LambdaMemoryMb` | `10240` | Lambda memory; Lambda allocates CPU proportionally. 10 GB recommended for 1080p. | | `LambdaTimeoutSec` | `900` | Per-invocation timeout. 15 min is Lambda's hard ceiling. | | `ReservedConcurrency` | `-1` | Hard cap on simultaneous Lambda invocations. `-1` = unreserved. Set to e.g. `50` to bound cost. | -| `ChromeSource` | `sparticuz` | Must match the `--source=` flag passed to `build-zip.ts`; a shell ZIP is over Lambda's limit. | +| `ChromeSource` | `sparticuz` | Only `sparticuz`: a ZIP built with `--source=chrome-headless-shell` is over Lambda's limit. | | `ChunkInvocationAlarmThreshold` | `1000` | CloudWatch alarm threshold (RenderChunk invocations per hour). | ## Cleanup diff --git a/examples/aws-lambda/template.yaml b/examples/aws-lambda/template.yaml index 0b4582c75f..3fcb7fc77f 100644 --- a/examples/aws-lambda/template.yaml +++ b/examples/aws-lambda/template.yaml @@ -49,13 +49,13 @@ Parameters: ChromeSource: Type: String Default: sparticuz - AllowedValues: [sparticuz, chrome-headless-shell] + AllowedValues: [sparticuz] Description: >- Which Chrome runtime the bundled ZIP was built with. Must match the `--source=` flag passed to `build-zip.ts`. The handler reads this - via the HYPERFRAMES_LAMBDA_CHROME_SOURCE env var at boot. A - chrome-headless-shell ZIP is over Lambda's unzipped limit today; the - arm64 video image always runs its own chrome-headless-shell. + via the HYPERFRAMES_LAMBDA_CHROME_SOURCE env var at boot. Only + sparticuz: a chrome-headless-shell ZIP is over Lambda's unzipped + limit, and the arm64 video image runs its own chrome-headless-shell. ChunkInvocationAlarmThreshold: Type: Number @@ -77,7 +77,6 @@ Parameters: Conditions: HasReservedConcurrency: !Not [!Equals [!Ref ReservedConcurrency, -1]] HasVideoFunction: !Not [!Equals [!Ref VideoImageUri, ""]] - BundlesChromeHeadlessShell: !Equals [!Ref ChromeSource, chrome-headless-shell] Globals: Function: @@ -97,11 +96,6 @@ Globals: Variables: NODE_OPTIONS: "--enable-source-maps" HYPERFRAMES_LAMBDA_CHROME_SOURCE: !Ref ChromeSource - # Where build-zip.ts --source=chrome-headless-shell puts the shell (BUNDLED_SHELL_PATH). - HYPERFRAMES_LAMBDA_CHROME_PATH: !If - - BundlesChromeHeadlessShell - - /var/task/chrome-headless-shell-linux64/chrome-headless-shell - - !Ref AWS::NoValue Resources: # ── S3 bucket for plan tarballs, chunk outputs, and final renders ─────── diff --git a/packages/aws-lambda/README.md b/packages/aws-lambda/README.md index a66d51c56e..b27bd9cfd0 100644 --- a/packages/aws-lambda/README.md +++ b/packages/aws-lambda/README.md @@ -104,13 +104,13 @@ bun run --cwd packages/aws-lambda build:zip -- --source=chrome-headless-shell The handler reads `HYPERFRAMES_LAMBDA_CHROME_SOURCE` at boot, and for `chrome-headless-shell` the executable from `HYPERFRAMES_LAMBDA_CHROME_PATH`. -`examples/aws-lambda/template.yaml` and the CDK construct set both from the -Chrome source. The build stages the repo's pinned shell build, and only when -its folder matches the pinned digest. It stages the whole folder (the +The build stages the repo's pinned shell build, and only when its folder +matches that build's pinned digest. It stages the whole folder (the executable needs its ICU data and `.pak` files beside it), which is over Lambda's 250 MiB unzipped limit for a ZIP function, so the build fails its -size gate and the x86 ZIP ships `@sparticuz/chromium`. `hyperframes lambda -deploy` refuses `--chrome-source chrome-headless-shell` for the same reason. +size gate and the x86 ZIP ships `@sparticuz/chromium`. For the same reason +`hyperframes lambda deploy`, `examples/aws-lambda/template.yaml` and the CDK +construct refuse `chrome-headless-shell` as the ZIP's Chrome source. The arm64 video image (`image/Dockerfile`, `video` target) does not use `@sparticuz/chromium`: it installs Chrome for Testing's linux-arm64 diff --git a/packages/aws-lambda/scripts/_chromeHeadlessShell.ts b/packages/aws-lambda/scripts/_chromeHeadlessShell.ts index 9fe04e2c91..bc59c8047b 100644 --- a/packages/aws-lambda/scripts/_chromeHeadlessShell.ts +++ b/packages/aws-lambda/scripts/_chromeHeadlessShell.ts @@ -1,8 +1,10 @@ import { cpSync, existsSync } from "node:fs"; -import { join } from "node:path"; +import { join, relative } from "node:path"; import { - isPinnedHeadlessShell, + CHROME_WRITES_WHILE_RUNNING, + headlessShellDigest, managedChromeVersion, + PINNED_SHELL_DIGESTS, } from "@hyperframes/engine/chrome-host-ceiling"; import { BUNDLED_SHELL_DIR } from "../src/chromium.js"; @@ -10,12 +12,13 @@ import { BUNDLED_SHELL_DIR } from "../src/chromium.js"; * Stage the repo's pinned linux64 chrome-headless-shell from the Puppeteer cache under `home` into * `//`. The whole folder: the executable alone exits at launch * ("Invalid file descriptor to ICU data received") without icudtl.dat and its .pak files. - * Only the pinned build, and only when its folder matches the pinned digest. Returns the version. + * Only the pinned build, and only when its folder matches that build's own pinned digest; the files the digest leaves + * out are not copied. Returns the version. */ export function stageChromeHeadlessShell( stagingDir: string, home = process.env.HOME ?? "/root", - isPinned: (executable: string) => boolean = isPinnedHeadlessShell, + pinned: Readonly> = PINNED_SHELL_DIGESTS, ): string { const version = managedChromeVersion("linux"); const cache = join(home, ".cache", "puppeteer"); @@ -33,13 +36,17 @@ export function stageChromeHeadlessShell( `before --source=chrome-headless-shell.`, ); } - if (!isPinned(executable)) { + const digest = headlessShellDigest(folder); + if (digest === undefined || !(pinned[version] ?? []).includes(digest)) { throw new Error( `[build-zip] ${folder} does not match the pinned chrome-headless-shell ${version} digest. ` + `Delete it and install it again.`, ); } // cpSync keeps each file's mode, so the executable stays executable. - cpSync(folder, join(stagingDir, BUNDLED_SHELL_DIR), { recursive: true }); + cpSync(folder, join(stagingDir, BUNDLED_SHELL_DIR), { + recursive: true, + filter: (path) => !CHROME_WRITES_WHILE_RUNNING.includes(relative(folder, path)), + }); return version; } diff --git a/packages/aws-lambda/scripts/build-zip.test.ts b/packages/aws-lambda/scripts/build-zip.test.ts index 1b40fd03a4..2f1f73fb87 100644 --- a/packages/aws-lambda/scripts/build-zip.test.ts +++ b/packages/aws-lambda/scripts/build-zip.test.ts @@ -12,9 +12,9 @@ import { import { tmpdir } from "node:os"; import { dirname, join } from "node:path"; import { pathToFileURL } from "node:url"; -import { managedChromeVersion } from "@hyperframes/engine/chrome-host-ceiling"; +import { headlessShellDigest, managedChromeVersion } from "@hyperframes/engine/chrome-host-ceiling"; import * as esbuild from "esbuild"; -import { BUNDLED_SHELL_DIR, BUNDLED_SHELL_PATH } from "../src/chromium.js"; +import { BUNDLED_SHELL_DIR } from "../src/chromium.js"; import { stageChromeHeadlessShell } from "./_chromeHeadlessShell.js"; import { HANDLER_BANNER } from "./_handlerBanner.js"; @@ -102,8 +102,15 @@ describe("build-zip chrome-headless-shell staging", () => { return home; } const pinned = managedChromeVersion("linux"); + const shellFolder = (home: string, version: string) => + join( + home, + ".cache/puppeteer/chrome-headless-shell", + `linux-${version}`, + "chrome-headless-shell-linux64", + ); - it("stages the whole pinned shell folder where the template and the CDK stack point", () => { + it("stages the whole pinned shell folder", () => { const home = puppeteerCache({ "999.0.0.1": { "chrome-headless-shell": "newer" }, [pinned]: { @@ -111,32 +118,37 @@ describe("build-zip chrome-headless-shell staging", () => { "icudtl.dat": "icu", "headless_lib_data.pak": "pak", "locales/en-US.pak": "locale", + "debug.log": "written by a run", }, }); const staging = mkdtempSync(join(tmpdir(), "hf-shell-staging-")); - const checked: string[] = []; try { - const isPinned = (exe: string) => checked.push(exe) > 0; - expect(stageChromeHeadlessShell(staging, home, isPinned)).toBe(pinned); - expect(checked).toEqual([ - join( - home, - ".cache/puppeteer/chrome-headless-shell", - `linux-${pinned}`, - "chrome-headless-shell-linux64", - "chrome-headless-shell", - ), - ]); + const digests = { [pinned]: [headlessShellDigest(shellFolder(home, pinned))!] }; + expect(stageChromeHeadlessShell(staging, home, digests)).toBe(pinned); const shell = join(staging, BUNDLED_SHELL_DIR); + // The digest leaves out what Chrome writes as it runs, so none of that ships. + expect(existsSync(join(shell, "debug.log"))).toBe(false); // Without icudtl.dat beside it the shell exits at launch, so the folder ships whole. expect(readFileSync(join(shell, "icudtl.dat"), "utf-8")).toBe("icu"); expect(readFileSync(join(shell, "headless_lib_data.pak"), "utf-8")).toBe("pak"); expect(readFileSync(join(shell, "locales/en-US.pak"), "utf-8")).toBe("locale"); expect(readFileSync(join(shell, "chrome-headless-shell"), "utf-8")).toBe("exe"); expect(statSync(join(shell, "chrome-headless-shell")).mode & 0o111).not.toBe(0); - expect(join("/var/task", BUNDLED_SHELL_DIR, "chrome-headless-shell")).toBe( - BUNDLED_SHELL_PATH, + } finally { + rmSync(home, { recursive: true, force: true }); + rmSync(staging, { recursive: true, force: true }); + } + }); + + it("refuses a folder whose digest is pinned only for another build", () => { + const home = puppeteerCache({ [pinned]: { "chrome-headless-shell": "exe" } }); + const staging = mkdtempSync(join(tmpdir(), "hf-shell-staging-")); + try { + const digests = { "150.0.7871.124": [headlessShellDigest(shellFolder(home, pinned))!] }; + expect(() => stageChromeHeadlessShell(staging, home, digests)).toThrow( + /does not match the pinned/, ); + expect(existsSync(join(staging, BUNDLED_SHELL_DIR))).toBe(false); } finally { rmSync(home, { recursive: true, force: true }); rmSync(staging, { recursive: true, force: true }); diff --git a/packages/aws-lambda/src/cdk/HyperframesRenderStack.snapshot.test.ts b/packages/aws-lambda/src/cdk/HyperframesRenderStack.snapshot.test.ts index 58f826ce08..f6d2c95b56 100644 --- a/packages/aws-lambda/src/cdk/HyperframesRenderStack.snapshot.test.ts +++ b/packages/aws-lambda/src/cdk/HyperframesRenderStack.snapshot.test.ts @@ -31,7 +31,6 @@ import { CONTENT_STORE_REFRESH_AFTER_DAYS, CONTENT_STORE_RETENTION_DAYS, } from "../contentStore.js"; -import { BUNDLED_SHELL_PATH } from "../chromium.js"; import { SITE_MANIFEST_INVALID } from "../siteManifest.js"; import { HyperframesRenderStack, @@ -411,44 +410,15 @@ describe("HyperframesRenderStack — snapshot", () => { ).toEqual([...new Set(render)].sort()); }); - it("points a chrome-headless-shell ZIP at its bundled shell, in CDK and SAM", () => { - const chromeEnv = (synthed: ReturnType) => { - const fn = Object.values(synthed.template.findResources("AWS::Lambda::Function"))[0] as { - Properties: { Environment: { Variables: Record } }; - }; - const vars = fn.Properties.Environment.Variables; - return [vars.HYPERFRAMES_LAMBDA_CHROME_SOURCE, vars.HYPERFRAMES_LAMBDA_CHROME_PATH]; - }; - expect(chromeEnv(SYNTHED)).toEqual(["sparticuz", undefined]); - expect(chromeEnv(doSynth(undefined, "chrome-headless-shell"))).toEqual([ - "chrome-headless-shell", - BUNDLED_SHELL_PATH, - ]); - - const globals = requireRecordProperty( - requireRecordProperty( - requireRecordProperty(readExampleTemplate("template.yaml"), "Globals", "SAM globals"), - "Function", - "SAM function globals", - ), - "Environment", - "SAM global environment", - ) as { Variables: Record }; - // The YAML parser drops the !If and !Ref tags and keeps [condition, then, else]. - expect(globals.Variables.HYPERFRAMES_LAMBDA_CHROME_PATH).toEqual([ - "BundlesChromeHeadlessShell", - BUNDLED_SHELL_PATH, - "AWS::NoValue", - ]); - const conditions = requireRecordProperty( - readExampleTemplate("template.yaml"), - "Conditions", - "SAM conditions", + it("refuses a chrome-headless-shell ZIP, which cannot fit a function, in CDK and SAM", () => { + expect(() => doSynth(undefined, "chrome-headless-shell")).toThrow(/does not fit a Lambda ZIP/); + const sam = readExampleTemplate("template.yaml"); + const parameters = requireRecordProperty(sam, "Parameters", "SAM parameters"); + const chromeSource = requireRecordProperty(parameters, "ChromeSource", "ChromeSource"); + expect(chromeSource.AllowedValues).toEqual(["sparticuz"]); + expect(envNames(sam, "Globals", "Function", "Environment", "Variables")).not.toContain( + "HYPERFRAMES_LAMBDA_CHROME_PATH", ); - expect(conditions.BundlesChromeHeadlessShell).toEqual([ - "ChromeSource", - "chrome-headless-shell", - ]); }, 30000); it("lets the render function abort the multipart uploads its S3 stitch starts", () => { diff --git a/packages/aws-lambda/src/cdk/HyperframesRenderStack.ts b/packages/aws-lambda/src/cdk/HyperframesRenderStack.ts index 76dbcc36d1..113424305b 100644 --- a/packages/aws-lambda/src/cdk/HyperframesRenderStack.ts +++ b/packages/aws-lambda/src/cdk/HyperframesRenderStack.ts @@ -30,7 +30,6 @@ import * as s3 from "aws-cdk-lib/aws-s3"; import * as sfn from "aws-cdk-lib/aws-stepfunctions"; import * as tasks from "aws-cdk-lib/aws-stepfunctions-tasks"; import { Construct } from "constructs"; -import { BUNDLED_SHELL_PATH } from "../chromium.js"; import { CONTENT_STORE_PREFIX, CONTENT_STORE_RETENTION_DAYS } from "../contentStore.js"; /** Construction-time props for {@link HyperframesRenderStack}. */ @@ -43,7 +42,7 @@ export interface HyperframesRenderStackProps { lambdaTimeoutSec?: number; /** Lambda reserved concurrency cap. `undefined` = unreserved (account default). */ reservedConcurrency?: number; - /** Which Chrome runtime was bundled into the handler ZIP. Default `"sparticuz"`; a shell ZIP is over Lambda's limit. */ + /** Which Chrome runtime was bundled into the handler ZIP. Only `"sparticuz"`: a shell ZIP is over Lambda's limit. */ chromeSource?: "sparticuz" | "chrome-headless-shell"; /** Threshold for the runaway-invocations alarm. Default 1000 invocations/hour. */ chunkInvocationAlarmThreshold?: number; @@ -63,15 +62,14 @@ const DEFAULT_TIMEOUT_SEC = 900; const DEFAULT_CHROME_SOURCE = "sparticuz"; const DEFAULT_ALARM_THRESHOLD = 1000; -/** The handler's Chrome source, and for the bundled shell, where build-zip.ts put it. */ -function chromeEnvironment( +function zipChromeSource( source: NonNullable, -): Record { - if (source === "sparticuz") return { HYPERFRAMES_LAMBDA_CHROME_SOURCE: source }; - return { - HYPERFRAMES_LAMBDA_CHROME_SOURCE: source, - HYPERFRAMES_LAMBDA_CHROME_PATH: BUNDLED_SHELL_PATH, - }; +): "sparticuz" { + if (source === "sparticuz") return source; + throw new Error( + "[HyperframesRenderStack] chromeSource chrome-headless-shell does not fit a Lambda ZIP function, so the " + + "handler ZIP ships @sparticuz/chromium. The arm64 video image runs chrome-headless-shell.", + ); } export class HyperframesRenderStack extends Construct { @@ -88,7 +86,7 @@ export class HyperframesRenderStack extends Construct { const projectName = props.projectName ?? "hyperframes"; const memorySize = props.lambdaMemoryMb ?? DEFAULT_MEMORY_MB; const timeoutSec = props.lambdaTimeoutSec ?? DEFAULT_TIMEOUT_SEC; - const chromeSource = props.chromeSource ?? DEFAULT_CHROME_SOURCE; + const chromeSource = zipChromeSource(props.chromeSource ?? DEFAULT_CHROME_SOURCE); const alarmThreshold = props.chunkInvocationAlarmThreshold ?? DEFAULT_ALARM_THRESHOLD; const handlerZipPath = props.handlerZipPath ?? defaultHandlerZipPath(); @@ -129,7 +127,7 @@ export class HyperframesRenderStack extends Construct { environment: { NODE_OPTIONS: "--enable-source-maps", TMPDIR: "/tmp", - ...chromeEnvironment(chromeSource), + HYPERFRAMES_LAMBDA_CHROME_SOURCE: chromeSource, HYPERFRAMES_RENDER_BUCKET: this.bucket.bucketName, HYPERFRAMES_LAMBDA_RESERVED_CONCURRENCY: String(props.reservedConcurrency ?? -1), HYPERFRAMES_STITCH_FUNCTION: `${projectName}-render`, diff --git a/packages/aws-lambda/src/chromium.ts b/packages/aws-lambda/src/chromium.ts index dd88a803be..dc2d561b44 100644 --- a/packages/aws-lambda/src/chromium.ts +++ b/packages/aws-lambda/src/chromium.ts @@ -39,7 +39,6 @@ import { existsSync } from "node:fs"; export type ChromeSource = "sparticuz" | "chrome-headless-shell"; export const BUNDLED_SHELL_DIR = "chrome-headless-shell-linux64"; -export const BUNDLED_SHELL_PATH = `/var/task/${BUNDLED_SHELL_DIR}/chrome-headless-shell`; /** * Thrown when the Chrome binary resolver can't produce a usable path. diff --git a/packages/engine/src/services/chromeHostCeiling.ts b/packages/engine/src/services/chromeHostCeiling.ts index a0fbd904f5..22f56eb13a 100644 --- a/packages/engine/src/services/chromeHostCeiling.ts +++ b/packages/engine/src/services/chromeHostCeiling.ts @@ -59,7 +59,7 @@ export const PINNED_SHELL_DIGESTS: Readonly> = "fc5a7041fc62b0740093077cb476cae232853a0b4272c6851add5f60ce11d1c6", ], }; -const CHROME_WRITES_WHILE_RUNNING = ["debug.log", "chrome_debug.log"]; +export const CHROME_WRITES_WHILE_RUNNING: readonly string[] = ["debug.log", "chrome_debug.log"]; function filesUnder(folder: string, prefix = ""): string[] | undefined { const names: string[] = []; diff --git a/scripts/package-size-budgets.json b/scripts/package-size-budgets.json index 40f362f1ec..f713725796 100644 --- a/scripts/package-size-budgets.json +++ b/scripts/package-size-budgets.json @@ -1,6 +1,6 @@ { "@hyperframes/aws-lambda": { - "packed": 80500, + "packed": 82000, "unpacked": 298000 }, "@hyperframes/cli": {