diff --git a/examples/aws-lambda/README.md b/examples/aws-lambda/README.md index 51e0f50d0b..210178d4d5 100644 --- a/examples/aws-lambda/README.md +++ b/examples/aws-lambda/README.md @@ -200,7 +200,7 @@ with `--profile ` or `AWS_PROFILE=`. | `LambdaMemoryMb` | `10240` | Lambda memory; Lambda allocates CPU proportionally. 10 GB recommended for 1080p. | | `LambdaTimeoutSec` | `900` | Per-invocation timeout. 15 min is Lambda's hard ceiling. | | `ReservedConcurrency` | `-1` | Hard cap on simultaneous Lambda invocations. `-1` = unreserved. Set to e.g. `50` to bound cost. | -| `ChromeSource` | `sparticuz` | Must match the `--source=` flag passed to `build-zip.ts`; a shell ZIP is over Lambda's limit. | +| `ChromeSource` | `sparticuz` | Only `sparticuz`: a ZIP built with `--source=chrome-headless-shell` is over Lambda's limit. | | `ChunkInvocationAlarmThreshold` | `1000` | CloudWatch alarm threshold (RenderChunk invocations per hour). | ## Cleanup diff --git a/examples/aws-lambda/template.yaml b/examples/aws-lambda/template.yaml index 0b4582c75f..3fcb7fc77f 100644 --- a/examples/aws-lambda/template.yaml +++ b/examples/aws-lambda/template.yaml @@ -49,13 +49,13 @@ Parameters: ChromeSource: Type: String Default: sparticuz - AllowedValues: [sparticuz, chrome-headless-shell] + AllowedValues: [sparticuz] Description: >- Which Chrome runtime the bundled ZIP was built with. Must match the `--source=` flag passed to `build-zip.ts`. The handler reads this - via the HYPERFRAMES_LAMBDA_CHROME_SOURCE env var at boot. A - chrome-headless-shell ZIP is over Lambda's unzipped limit today; the - arm64 video image always runs its own chrome-headless-shell. + via the HYPERFRAMES_LAMBDA_CHROME_SOURCE env var at boot. Only + sparticuz: a chrome-headless-shell ZIP is over Lambda's unzipped + limit, and the arm64 video image runs its own chrome-headless-shell. ChunkInvocationAlarmThreshold: Type: Number @@ -77,7 +77,6 @@ Parameters: Conditions: HasReservedConcurrency: !Not [!Equals [!Ref ReservedConcurrency, -1]] HasVideoFunction: !Not [!Equals [!Ref VideoImageUri, ""]] - BundlesChromeHeadlessShell: !Equals [!Ref ChromeSource, chrome-headless-shell] Globals: Function: @@ -97,11 +96,6 @@ Globals: Variables: NODE_OPTIONS: "--enable-source-maps" HYPERFRAMES_LAMBDA_CHROME_SOURCE: !Ref ChromeSource - # Where build-zip.ts --source=chrome-headless-shell puts the shell (BUNDLED_SHELL_PATH). - HYPERFRAMES_LAMBDA_CHROME_PATH: !If - - BundlesChromeHeadlessShell - - /var/task/chrome-headless-shell-linux64/chrome-headless-shell - - !Ref AWS::NoValue Resources: # ── S3 bucket for plan tarballs, chunk outputs, and final renders ─────── diff --git a/packages/aws-lambda/README.md b/packages/aws-lambda/README.md index a66d51c56e..b27bd9cfd0 100644 --- a/packages/aws-lambda/README.md +++ b/packages/aws-lambda/README.md @@ -104,13 +104,13 @@ bun run --cwd packages/aws-lambda build:zip -- --source=chrome-headless-shell The handler reads `HYPERFRAMES_LAMBDA_CHROME_SOURCE` at boot, and for `chrome-headless-shell` the executable from `HYPERFRAMES_LAMBDA_CHROME_PATH`. -`examples/aws-lambda/template.yaml` and the CDK construct set both from the -Chrome source. The build stages the repo's pinned shell build, and only when -its folder matches the pinned digest. It stages the whole folder (the +The build stages the repo's pinned shell build, and only when its folder +matches that build's pinned digest. It stages the whole folder (the executable needs its ICU data and `.pak` files beside it), which is over Lambda's 250 MiB unzipped limit for a ZIP function, so the build fails its -size gate and the x86 ZIP ships `@sparticuz/chromium`. `hyperframes lambda -deploy` refuses `--chrome-source chrome-headless-shell` for the same reason. +size gate and the x86 ZIP ships `@sparticuz/chromium`. For the same reason +`hyperframes lambda deploy`, `examples/aws-lambda/template.yaml` and the CDK +construct refuse `chrome-headless-shell` as the ZIP's Chrome source. The arm64 video image (`image/Dockerfile`, `video` target) does not use `@sparticuz/chromium`: it installs Chrome for Testing's linux-arm64 diff --git a/packages/aws-lambda/scripts/_chromeHeadlessShell.ts b/packages/aws-lambda/scripts/_chromeHeadlessShell.ts index 9fe04e2c91..bc59c8047b 100644 --- a/packages/aws-lambda/scripts/_chromeHeadlessShell.ts +++ b/packages/aws-lambda/scripts/_chromeHeadlessShell.ts @@ -1,8 +1,10 @@ import { cpSync, existsSync } from "node:fs"; -import { join } from "node:path"; +import { join, relative } from "node:path"; import { - isPinnedHeadlessShell, + CHROME_WRITES_WHILE_RUNNING, + headlessShellDigest, managedChromeVersion, + PINNED_SHELL_DIGESTS, } from "@hyperframes/engine/chrome-host-ceiling"; import { BUNDLED_SHELL_DIR } from "../src/chromium.js"; @@ -10,12 +12,13 @@ import { BUNDLED_SHELL_DIR } from "../src/chromium.js"; * Stage the repo's pinned linux64 chrome-headless-shell from the Puppeteer cache under `home` into * `//`. The whole folder: the executable alone exits at launch * ("Invalid file descriptor to ICU data received") without icudtl.dat and its .pak files. - * Only the pinned build, and only when its folder matches the pinned digest. Returns the version. + * Only the pinned build, and only when its folder matches that build's own pinned digest; the files the digest leaves + * out are not copied. Returns the version. */ export function stageChromeHeadlessShell( stagingDir: string, home = process.env.HOME ?? "/root", - isPinned: (executable: string) => boolean = isPinnedHeadlessShell, + pinned: Readonly> = PINNED_SHELL_DIGESTS, ): string { const version = managedChromeVersion("linux"); const cache = join(home, ".cache", "puppeteer"); @@ -33,13 +36,17 @@ export function stageChromeHeadlessShell( `before --source=chrome-headless-shell.`, ); } - if (!isPinned(executable)) { + const digest = headlessShellDigest(folder); + if (digest === undefined || !(pinned[version] ?? []).includes(digest)) { throw new Error( `[build-zip] ${folder} does not match the pinned chrome-headless-shell ${version} digest. ` + `Delete it and install it again.`, ); } // cpSync keeps each file's mode, so the executable stays executable. - cpSync(folder, join(stagingDir, BUNDLED_SHELL_DIR), { recursive: true }); + cpSync(folder, join(stagingDir, BUNDLED_SHELL_DIR), { + recursive: true, + filter: (path) => !CHROME_WRITES_WHILE_RUNNING.includes(relative(folder, path)), + }); return version; } diff --git a/packages/aws-lambda/scripts/build-zip.test.ts b/packages/aws-lambda/scripts/build-zip.test.ts index 1b40fd03a4..2f1f73fb87 100644 --- a/packages/aws-lambda/scripts/build-zip.test.ts +++ b/packages/aws-lambda/scripts/build-zip.test.ts @@ -12,9 +12,9 @@ import { import { tmpdir } from "node:os"; import { dirname, join } from "node:path"; import { pathToFileURL } from "node:url"; -import { managedChromeVersion } from "@hyperframes/engine/chrome-host-ceiling"; +import { headlessShellDigest, managedChromeVersion } from "@hyperframes/engine/chrome-host-ceiling"; import * as esbuild from "esbuild"; -import { BUNDLED_SHELL_DIR, BUNDLED_SHELL_PATH } from "../src/chromium.js"; +import { BUNDLED_SHELL_DIR } from "../src/chromium.js"; import { stageChromeHeadlessShell } from "./_chromeHeadlessShell.js"; import { HANDLER_BANNER } from "./_handlerBanner.js"; @@ -102,8 +102,15 @@ describe("build-zip chrome-headless-shell staging", () => { return home; } const pinned = managedChromeVersion("linux"); + const shellFolder = (home: string, version: string) => + join( + home, + ".cache/puppeteer/chrome-headless-shell", + `linux-${version}`, + "chrome-headless-shell-linux64", + ); - it("stages the whole pinned shell folder where the template and the CDK stack point", () => { + it("stages the whole pinned shell folder", () => { const home = puppeteerCache({ "999.0.0.1": { "chrome-headless-shell": "newer" }, [pinned]: { @@ -111,32 +118,37 @@ describe("build-zip chrome-headless-shell staging", () => { "icudtl.dat": "icu", "headless_lib_data.pak": "pak", "locales/en-US.pak": "locale", + "debug.log": "written by a run", }, }); const staging = mkdtempSync(join(tmpdir(), "hf-shell-staging-")); - const checked: string[] = []; try { - const isPinned = (exe: string) => checked.push(exe) > 0; - expect(stageChromeHeadlessShell(staging, home, isPinned)).toBe(pinned); - expect(checked).toEqual([ - join( - home, - ".cache/puppeteer/chrome-headless-shell", - `linux-${pinned}`, - "chrome-headless-shell-linux64", - "chrome-headless-shell", - ), - ]); + const digests = { [pinned]: [headlessShellDigest(shellFolder(home, pinned))!] }; + expect(stageChromeHeadlessShell(staging, home, digests)).toBe(pinned); const shell = join(staging, BUNDLED_SHELL_DIR); + // The digest leaves out what Chrome writes as it runs, so none of that ships. + expect(existsSync(join(shell, "debug.log"))).toBe(false); // Without icudtl.dat beside it the shell exits at launch, so the folder ships whole. expect(readFileSync(join(shell, "icudtl.dat"), "utf-8")).toBe("icu"); expect(readFileSync(join(shell, "headless_lib_data.pak"), "utf-8")).toBe("pak"); expect(readFileSync(join(shell, "locales/en-US.pak"), "utf-8")).toBe("locale"); expect(readFileSync(join(shell, "chrome-headless-shell"), "utf-8")).toBe("exe"); expect(statSync(join(shell, "chrome-headless-shell")).mode & 0o111).not.toBe(0); - expect(join("/var/task", BUNDLED_SHELL_DIR, "chrome-headless-shell")).toBe( - BUNDLED_SHELL_PATH, + } finally { + rmSync(home, { recursive: true, force: true }); + rmSync(staging, { recursive: true, force: true }); + } + }); + + it("refuses a folder whose digest is pinned only for another build", () => { + const home = puppeteerCache({ [pinned]: { "chrome-headless-shell": "exe" } }); + const staging = mkdtempSync(join(tmpdir(), "hf-shell-staging-")); + try { + const digests = { "150.0.7871.124": [headlessShellDigest(shellFolder(home, pinned))!] }; + expect(() => stageChromeHeadlessShell(staging, home, digests)).toThrow( + /does not match the pinned/, ); + expect(existsSync(join(staging, BUNDLED_SHELL_DIR))).toBe(false); } finally { rmSync(home, { recursive: true, force: true }); rmSync(staging, { recursive: true, force: true }); diff --git a/packages/aws-lambda/src/cdk/HyperframesRenderStack.snapshot.test.ts b/packages/aws-lambda/src/cdk/HyperframesRenderStack.snapshot.test.ts index 58f826ce08..f6d2c95b56 100644 --- a/packages/aws-lambda/src/cdk/HyperframesRenderStack.snapshot.test.ts +++ b/packages/aws-lambda/src/cdk/HyperframesRenderStack.snapshot.test.ts @@ -31,7 +31,6 @@ import { CONTENT_STORE_REFRESH_AFTER_DAYS, CONTENT_STORE_RETENTION_DAYS, } from "../contentStore.js"; -import { BUNDLED_SHELL_PATH } from "../chromium.js"; import { SITE_MANIFEST_INVALID } from "../siteManifest.js"; import { HyperframesRenderStack, @@ -411,44 +410,15 @@ describe("HyperframesRenderStack — snapshot", () => { ).toEqual([...new Set(render)].sort()); }); - it("points a chrome-headless-shell ZIP at its bundled shell, in CDK and SAM", () => { - const chromeEnv = (synthed: ReturnType) => { - const fn = Object.values(synthed.template.findResources("AWS::Lambda::Function"))[0] as { - Properties: { Environment: { Variables: Record } }; - }; - const vars = fn.Properties.Environment.Variables; - return [vars.HYPERFRAMES_LAMBDA_CHROME_SOURCE, vars.HYPERFRAMES_LAMBDA_CHROME_PATH]; - }; - expect(chromeEnv(SYNTHED)).toEqual(["sparticuz", undefined]); - expect(chromeEnv(doSynth(undefined, "chrome-headless-shell"))).toEqual([ - "chrome-headless-shell", - BUNDLED_SHELL_PATH, - ]); - - const globals = requireRecordProperty( - requireRecordProperty( - requireRecordProperty(readExampleTemplate("template.yaml"), "Globals", "SAM globals"), - "Function", - "SAM function globals", - ), - "Environment", - "SAM global environment", - ) as { Variables: Record }; - // The YAML parser drops the !If and !Ref tags and keeps [condition, then, else]. - expect(globals.Variables.HYPERFRAMES_LAMBDA_CHROME_PATH).toEqual([ - "BundlesChromeHeadlessShell", - BUNDLED_SHELL_PATH, - "AWS::NoValue", - ]); - const conditions = requireRecordProperty( - readExampleTemplate("template.yaml"), - "Conditions", - "SAM conditions", + it("refuses a chrome-headless-shell ZIP, which cannot fit a function, in CDK and SAM", () => { + expect(() => doSynth(undefined, "chrome-headless-shell")).toThrow(/does not fit a Lambda ZIP/); + const sam = readExampleTemplate("template.yaml"); + const parameters = requireRecordProperty(sam, "Parameters", "SAM parameters"); + const chromeSource = requireRecordProperty(parameters, "ChromeSource", "ChromeSource"); + expect(chromeSource.AllowedValues).toEqual(["sparticuz"]); + expect(envNames(sam, "Globals", "Function", "Environment", "Variables")).not.toContain( + "HYPERFRAMES_LAMBDA_CHROME_PATH", ); - expect(conditions.BundlesChromeHeadlessShell).toEqual([ - "ChromeSource", - "chrome-headless-shell", - ]); }, 30000); it("lets the render function abort the multipart uploads its S3 stitch starts", () => { diff --git a/packages/aws-lambda/src/cdk/HyperframesRenderStack.ts b/packages/aws-lambda/src/cdk/HyperframesRenderStack.ts index 76dbcc36d1..113424305b 100644 --- a/packages/aws-lambda/src/cdk/HyperframesRenderStack.ts +++ b/packages/aws-lambda/src/cdk/HyperframesRenderStack.ts @@ -30,7 +30,6 @@ import * as s3 from "aws-cdk-lib/aws-s3"; import * as sfn from "aws-cdk-lib/aws-stepfunctions"; import * as tasks from "aws-cdk-lib/aws-stepfunctions-tasks"; import { Construct } from "constructs"; -import { BUNDLED_SHELL_PATH } from "../chromium.js"; import { CONTENT_STORE_PREFIX, CONTENT_STORE_RETENTION_DAYS } from "../contentStore.js"; /** Construction-time props for {@link HyperframesRenderStack}. */ @@ -43,7 +42,7 @@ export interface HyperframesRenderStackProps { lambdaTimeoutSec?: number; /** Lambda reserved concurrency cap. `undefined` = unreserved (account default). */ reservedConcurrency?: number; - /** Which Chrome runtime was bundled into the handler ZIP. Default `"sparticuz"`; a shell ZIP is over Lambda's limit. */ + /** Which Chrome runtime was bundled into the handler ZIP. Only `"sparticuz"`: a shell ZIP is over Lambda's limit. */ chromeSource?: "sparticuz" | "chrome-headless-shell"; /** Threshold for the runaway-invocations alarm. Default 1000 invocations/hour. */ chunkInvocationAlarmThreshold?: number; @@ -63,15 +62,14 @@ const DEFAULT_TIMEOUT_SEC = 900; const DEFAULT_CHROME_SOURCE = "sparticuz"; const DEFAULT_ALARM_THRESHOLD = 1000; -/** The handler's Chrome source, and for the bundled shell, where build-zip.ts put it. */ -function chromeEnvironment( +function zipChromeSource( source: NonNullable, -): Record { - if (source === "sparticuz") return { HYPERFRAMES_LAMBDA_CHROME_SOURCE: source }; - return { - HYPERFRAMES_LAMBDA_CHROME_SOURCE: source, - HYPERFRAMES_LAMBDA_CHROME_PATH: BUNDLED_SHELL_PATH, - }; +): "sparticuz" { + if (source === "sparticuz") return source; + throw new Error( + "[HyperframesRenderStack] chromeSource chrome-headless-shell does not fit a Lambda ZIP function, so the " + + "handler ZIP ships @sparticuz/chromium. The arm64 video image runs chrome-headless-shell.", + ); } export class HyperframesRenderStack extends Construct { @@ -88,7 +86,7 @@ export class HyperframesRenderStack extends Construct { const projectName = props.projectName ?? "hyperframes"; const memorySize = props.lambdaMemoryMb ?? DEFAULT_MEMORY_MB; const timeoutSec = props.lambdaTimeoutSec ?? DEFAULT_TIMEOUT_SEC; - const chromeSource = props.chromeSource ?? DEFAULT_CHROME_SOURCE; + const chromeSource = zipChromeSource(props.chromeSource ?? DEFAULT_CHROME_SOURCE); const alarmThreshold = props.chunkInvocationAlarmThreshold ?? DEFAULT_ALARM_THRESHOLD; const handlerZipPath = props.handlerZipPath ?? defaultHandlerZipPath(); @@ -129,7 +127,7 @@ export class HyperframesRenderStack extends Construct { environment: { NODE_OPTIONS: "--enable-source-maps", TMPDIR: "/tmp", - ...chromeEnvironment(chromeSource), + HYPERFRAMES_LAMBDA_CHROME_SOURCE: chromeSource, HYPERFRAMES_RENDER_BUCKET: this.bucket.bucketName, HYPERFRAMES_LAMBDA_RESERVED_CONCURRENCY: String(props.reservedConcurrency ?? -1), HYPERFRAMES_STITCH_FUNCTION: `${projectName}-render`, diff --git a/packages/aws-lambda/src/chromium.ts b/packages/aws-lambda/src/chromium.ts index dd88a803be..dc2d561b44 100644 --- a/packages/aws-lambda/src/chromium.ts +++ b/packages/aws-lambda/src/chromium.ts @@ -39,7 +39,6 @@ import { existsSync } from "node:fs"; export type ChromeSource = "sparticuz" | "chrome-headless-shell"; export const BUNDLED_SHELL_DIR = "chrome-headless-shell-linux64"; -export const BUNDLED_SHELL_PATH = `/var/task/${BUNDLED_SHELL_DIR}/chrome-headless-shell`; /** * Thrown when the Chrome binary resolver can't produce a usable path. diff --git a/packages/engine/src/services/chromeHostCeiling.ts b/packages/engine/src/services/chromeHostCeiling.ts index a0fbd904f5..22f56eb13a 100644 --- a/packages/engine/src/services/chromeHostCeiling.ts +++ b/packages/engine/src/services/chromeHostCeiling.ts @@ -59,7 +59,7 @@ export const PINNED_SHELL_DIGESTS: Readonly> = "fc5a7041fc62b0740093077cb476cae232853a0b4272c6851add5f60ce11d1c6", ], }; -const CHROME_WRITES_WHILE_RUNNING = ["debug.log", "chrome_debug.log"]; +export const CHROME_WRITES_WHILE_RUNNING: readonly string[] = ["debug.log", "chrome_debug.log"]; function filesUnder(folder: string, prefix = ""): string[] | undefined { const names: string[] = []; diff --git a/scripts/package-size-budgets.json b/scripts/package-size-budgets.json index 40f362f1ec..f713725796 100644 --- a/scripts/package-size-budgets.json +++ b/scripts/package-size-budgets.json @@ -1,6 +1,6 @@ { "@hyperframes/aws-lambda": { - "packed": 80500, + "packed": 82000, "unpacked": 298000 }, "@hyperframes/cli": {