Files
hyperframes/packages/aws-lambda/scripts/build-zip.ts
T
James RussoandClaude Opus 5.5 81d73d2278 fix(aws-lambda): chrome-headless-shell 158 in the arm64 video image, sparticuz 153 in the x86 ZIP (#8155)
* fix(aws-lambda): run chrome-headless-shell 158 in the arm64 video image

The arm64 video image now installs Chrome for Testing's linux-arm64 chrome-headless-shell
158.0.8127.36, pinned by sha256 and fetched with curl -f, as the whole shell folder. The Lambda
Node 22 base has none of the shared libraries the shell links and no fonts, so the image installs
them from the base's repository at the same pinned release. The image sets
HYPERFRAMES_LAMBDA_CHROME_SOURCE and HYPERFRAMES_LAMBDA_CHROME_PATH itself, and the nested video
function no longer passes ChromeSource, which would override them. stage-arm64-image.ts stops
fetching the sparticuz arm64 pack and drops the ZIP's x86 sparticuz Chrome instead.

The x86 handler ZIP stays on @sparticuz/chromium, bumped from 148.0.0 to 153.0.0. It builds at
258,502,222 bytes unzipped, under Lambda's 262,144,000-byte limit and the 248 MiB in-house gate.

Two fixes for --source=chrome-headless-shell:
- build-zip.ts staged only the executable, which exits at launch without its ICU data and .pak
  files. It now stages the whole shell folder at chrome-headless-shell-linux64/.
- Nothing set HYPERFRAMES_LAMBDA_CHROME_PATH, which src/chromium.ts requires for that source.
  The SAM template and the CDK stack now set it to the bundled shell when ChromeSource asks for it.

Drops the unused @types/aws-lambda devDependency, which the dead-code audit reports whenever this
package's manifest changes.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* ci(publish): build the Lambda arm64 video image on an ARM runner

The video stage now installs Chrome's system libraries with microdnf, so it runs code for arm64.
The pack job built that image on ubuntu-latest with no emulation, which it could do only while the
stage copied files. A new lambda-video-image job builds it natively on ubuntu-24.04-arm, from the
handler zip whose sha256 the zip job reported, and reports the OCI tarball's sha256. Pack downloads
that tarball, checks the sha256 against the job's, and pins it as before.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* chore(aws-lambda): fit the comment ratchet and the package size budget

chromium.ts loses the two doc lines on BUNDLED_SHELL_DIR and BUNDLED_SHELL_PATH; the names and the
SAM template's comment already say what they are.

@hyperframes/aws-lambda's unpacked budget goes from 290,508 to 298,000 bytes. The package unpacks to
295,324 bytes, up from the shared chrome-headless-shell staging module the ZIP and image scripts now
import and from the README and docs that describe the two Chrome sources.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-18 09:07:46 +00:00

438 lines
18 KiB
TypeScript

#!/usr/bin/env tsx
/**
* Build the AWS Lambda deployment ZIP.
*
* Pack layout (paths inside the ZIP are relative to Lambda's
* `/var/task/`):
*
* handler.mjs — bundled entry, set as Lambda's Handler
* handler.mjs.map — sourcemap (debugging aid; small)
* bin/ffmpeg — ffmpeg-static binary
* chrome-headless-shell-linux64/
* — fallback Chrome, the whole shell folder (only
* with --source=chrome-headless-shell)
* node_modules/@sparticuz/chromium/
* — primary Chrome (lives under node_modules so
* runtime `import("@sparticuz/chromium")`
* resolves; the package's own tarball stays
* inside).
*
* The handler bundle (esbuild) externalises modules whose binary assets
* must be present at runtime — `@sparticuz/chromium` for its bin tarball,
* `puppeteer-core` because Lambda runtime resolves it via Node module
* resolution from `node_modules/`. Everything else is inlined for cold
* start speed.
*
* Run:
* bun run --cwd packages/aws-lambda build:zip
* bun run --cwd packages/aws-lambda build:zip -- --source=chrome-headless-shell
*
* Outputs the resolved ZIP path + size to stdout and writes a sidecar
* JSON (`dist/handler.zip.manifest.json`) describing the contents.
*/
import { spawnSync } from "node:child_process";
import {
chmodSync,
closeSync,
copyFileSync,
cpSync,
existsSync,
mkdirSync,
openSync,
readdirSync,
readSync,
rmSync,
statSync,
writeFileSync,
} from "node:fs";
import { dirname, join, resolve } from "node:path";
import { fileURLToPath } from "node:url";
import * as esbuild from "esbuild";
import { BUNDLED_SHELL_DIR } from "../src/chromium.js";
import { stageChromeHeadlessShell } from "./_chromeHeadlessShell.js";
import { formatBytes } from "./_formatBytes.js";
import { HANDLER_BANNER } from "./_handlerBanner.js";
import { RUNTIME_LOCK_DIR, assertRuntimeLockCurrent } from "./runtimeLock.js";
const scriptDir = dirname(fileURLToPath(import.meta.url));
const packageRoot = resolve(scriptDir, "..");
const monorepoRoot = resolve(packageRoot, "../..");
const distDir = join(packageRoot, "dist");
interface BuildOptions {
source: "sparticuz" | "chrome-headless-shell";
/** Hard upper bound on the unzipped bundle size in bytes (Lambda limit is 250 MiB). */
maxUnzippedBytes: number;
/** Hard upper bound on the ZIP file size in bytes. */
maxZippedBytes: number;
}
const DEFAULT_OPTIONS: BuildOptions = {
source: "sparticuz",
// Lambda's hard ceiling for ZIP-deployed functions is 250 MiB unzipped
// (AWS docs label it "250 MB" but the 262144000-byte value is 250
// binary mebibytes). We gate at 248 MiB to keep ~2 MiB of headroom —
// the sparticuz Chrome (~70 MiB) + ffmpeg (~80 MiB) + ffprobe (~62
// MiB) + bundled Node deps put us close to the ceiling. Chrome itself
// decompresses into Lambda's `/tmp` at cold start, which has its own
// 10 GiB budget, so the unzipped /var/task footprint above is what
// actually competes with Lambda's 250 MiB limit.
maxUnzippedBytes: 248 * 1024 * 1024,
// Lambda's only zipped-size cap is for direct console/CLI uploads (50
// MiB); S3-deployed functions are bounded by the unzipped ceiling. We
// gate at 150 MiB to flag a sudden bundle-size regression without
// false-failing on the natural ~100 MiB sparticuz + ffmpeg payload.
maxZippedBytes: 150 * 1024 * 1024,
};
function parseArgs(argv: string[]): BuildOptions {
const opts = { ...DEFAULT_OPTIONS };
for (const arg of argv.slice(2)) {
if (arg.startsWith("--source=")) {
const v = arg.slice("--source=".length);
if (v !== "sparticuz" && v !== "chrome-headless-shell") {
throw new Error(`--source must be 'sparticuz' or 'chrome-headless-shell' (got ${v})`);
}
opts.source = v;
} else if (arg.startsWith("--max-unzipped=")) {
opts.maxUnzippedBytes = Number.parseInt(arg.slice("--max-unzipped=".length), 10);
} else if (arg.startsWith("--max-zipped=")) {
opts.maxZippedBytes = Number.parseInt(arg.slice("--max-zipped=".length), 10);
} else if (arg === "--help") {
console.log(
"Usage: tsx build-zip.ts [--source=sparticuz|chrome-headless-shell]\n" +
" [--max-unzipped=<bytes>] [--max-zipped=<bytes>]",
);
process.exit(0);
} else {
throw new Error(`Unknown flag: ${arg}`);
}
}
return opts;
}
async function main(): Promise<void> {
const opts = parseArgs(process.argv);
const start = Date.now();
// Leave the package build in dist/: the release job zips after `bun run build`.
const stagingDir = join(distDir, "staging");
rmSync(stagingDir, { recursive: true, force: true });
rmSync(join(distDir, "handler.zip"), { force: true });
mkdirSync(stagingDir, { recursive: true });
console.log(`[build-zip] source=${opts.source}`);
// 1. Bundle the handler.
await bundleHandler(stagingDir);
// 2. Stage runtime modules (puppeteer-core + @sparticuz/chromium or the
// fallback chrome-headless-shell tar).
stageRuntimeModules(stagingDir, opts.source);
// 3. Stage the ffmpeg binary.
stageFfmpeg(stagingDir);
// 3b. Stage the hyperframe runtime manifest + IIFE as siblings of
// handler.mjs. The producer's `hyperframeRuntimeLoader` checks
// SIBLING_MANIFEST_PATH first, so dropping the manifest alongside
// the bundled handler at /var/task/hyperframe.manifest.json lets
// renderChunk find it without needing PRODUCER_HYPERFRAME_MANIFEST_PATH.
stageHyperframeRuntime(stagingDir);
// 4. If we're on the chrome-headless-shell fallback, stage that binary.
if (opts.source === "chrome-headless-shell") {
const version = stageChromeHeadlessShell(stagingDir);
console.log(`[build-zip] staged chrome-headless-shell (${version}) → ${BUNDLED_SHELL_DIR}/`);
}
// 5. Compute the unzipped size BEFORE zipping so we fail loud when over budget.
const unzippedBytes = directorySizeBytes(stagingDir);
console.log(`[build-zip] unzipped staging size: ${formatBytes(unzippedBytes)}`);
if (unzippedBytes > opts.maxUnzippedBytes) {
throw new Error(
`[build-zip] unzipped bundle ${formatBytes(unzippedBytes)} exceeds limit ${formatBytes(
opts.maxUnzippedBytes,
)} (Lambda ZIP ceiling: 250 MiB unzipped). ` +
`Switch --source to the lighter option, or move Chrome to a Lambda Layer.`,
);
}
// 6. Build the ZIP.
const zipPath = join(distDir, "handler.zip");
zipDirectory(stagingDir, zipPath);
const zippedBytes = statSync(zipPath).size;
console.log(`[build-zip] zip size: ${formatBytes(zippedBytes)} → ${zipPath}`);
if (zippedBytes > opts.maxZippedBytes) {
throw new Error(
`[build-zip] zip ${formatBytes(zippedBytes)} exceeds ZIP size limit ${formatBytes(
opts.maxZippedBytes,
)}.`,
);
}
// 7. Sidecar manifest.
const manifest = {
builtAt: new Date().toISOString(),
durationMs: Date.now() - start,
source: opts.source,
unzippedBytes,
zippedBytes,
maxUnzippedBytes: opts.maxUnzippedBytes,
maxZippedBytes: opts.maxZippedBytes,
};
writeFileSync(join(distDir, "handler.zip.manifest.json"), JSON.stringify(manifest, null, 2));
// 8. Cleanup staging.
rmSync(stagingDir, { recursive: true, force: true });
console.log(`[build-zip] done in ${Date.now() - start}ms`);
}
async function bundleHandler(stagingDir: string): Promise<void> {
const entry = join(packageRoot, "src/handler.ts");
const outfile = join(stagingDir, "handler.mjs");
const workspaceAliasPlugin: esbuild.Plugin = {
name: "workspace-alias",
setup(build) {
build.onResolve({ filter: /^@hyperframes\/producer\/distributed$/ }, () => ({
path: resolve(monorepoRoot, "packages/producer/src/distributed.ts"),
}));
build.onResolve({ filter: /^@hyperframes\/producer$/ }, () => ({
path: resolve(monorepoRoot, "packages/producer/src/index.ts"),
}));
build.onResolve({ filter: /^@hyperframes\/engine$/ }, () => ({
path: resolve(monorepoRoot, "packages/engine/src/index.ts"),
}));
build.onResolve({ filter: /^@hyperframes\/engine\/alpha-blit$/ }, () => ({
path: resolve(monorepoRoot, "packages/engine/src/utils/alphaBlit.ts"),
}));
build.onResolve({ filter: /^@hyperframes\/engine\/shader-transitions$/ }, () => ({
path: resolve(monorepoRoot, "packages/engine/src/utils/shaderTransitions.ts"),
}));
build.onResolve({ filter: /^@hyperframes\/core$/ }, () => ({
path: resolve(monorepoRoot, "packages/core/src/index.ts"),
}));
build.onResolve({ filter: /^@hyperframes\/core\/lint$/ }, () => ({
path: resolve(monorepoRoot, "packages/core/src/lint/index.ts"),
}));
},
};
await esbuild.build({
bundle: true,
platform: "node",
target: "node22",
format: "esm",
// Externalise binary-shipped modules so node module resolution picks
// them up at runtime. esbuild would otherwise try to inline their
// postinstall-extracted binaries, which it cannot do.
external: [
"@sparticuz/chromium",
"puppeteer-core",
"puppeteer",
// AWS SDK v3 is pre-installed in the Lambda Node 22 runtime; mark
// external so we don't double-bundle 3+ MiB of SDK.
"@aws-sdk/client-lambda",
"@aws-sdk/client-s3",
],
plugins: [workspaceAliasPlugin],
minify: false,
// sourcemap=false: the ZIP is tight on Lambda's 250 MiB unzipped cap
// (Chrome ~70 MiB + ffmpeg ~80 MiB + ffprobe ~62 MiB + Node deps). A
// 4-5 MiB sourcemap puts us over. Re-enable for local debugging by passing
// --sourcemap; the bundle's stack traces stay readable enough without
// it because we don't minify.
sourcemap: false,
entryPoints: [entry],
outfile,
// See HANDLER_BANNER (_handlerBanner.ts) for why the ESM bundle needs the
// CJS require/__filename/__dirname shims.
banner: {
js: HANDLER_BANNER,
},
});
console.log(`[build-zip] bundled handler → ${outfile}`);
}
function stageRuntimeModules(stagingDir: string, source: BuildOptions["source"]): void {
// This runs in the publish job: node_modules come only from the committed lockfile, with
// lifecycle scripts off, and in the repo its direct versions must match bun.lock.
assertRuntimeLockCurrent(source);
const lockDir = join(RUNTIME_LOCK_DIR, source);
copyFileSync(join(lockDir, "package.json"), join(stagingDir, "package.json"));
copyFileSync(join(lockDir, "package-lock.json"), join(stagingDir, "package-lock.json"));
const result = spawnSync(
"npm",
["ci", "--ignore-scripts", "--no-audit", "--no-fund", "--omit=dev", "--omit=optional"],
{ cwd: stagingDir, stdio: "inherit" },
);
if (result.status !== 0) {
throw new Error(`[build-zip] npm ci into staging failed (status ${result.status})`);
}
rmSync(join(stagingDir, "package-lock.json"));
console.log(`[build-zip] staged node_modules via npm ci from runtime-lock/${source}`);
}
function resolveModuleDir(moduleName: string): string {
// Walk up from packageRoot to find a matching node_modules entry.
// Used by stageFfmpeg below; the @sparticuz/chromium + puppeteer-core
// paths now go through npm install instead.
let dir = packageRoot;
for (let i = 0; i < 5; i++) {
const candidate = join(dir, "node_modules", moduleName);
if (existsSync(candidate)) return candidate;
dir = dirname(dir);
}
throw new Error(
`[build-zip] could not resolve ${moduleName} from ${packageRoot} — run 'bun install' first.`,
);
}
function stageHyperframeRuntime(stagingDir: string): void {
const coreDist = resolve(monorepoRoot, "packages/core/dist");
const manifestSrc = join(coreDist, "hyperframe.manifest.json");
const iifeSrc = join(coreDist, "hyperframe.runtime.iife.js");
if (!existsSync(manifestSrc) || !existsSync(iifeSrc)) {
throw new Error(
`[build-zip] hyperframe runtime artifacts missing under ${coreDist}. ` +
`Run 'bun run --filter @hyperframes/core build:hyperframes-runtime:modular' first.`,
);
}
cpSync(manifestSrc, join(stagingDir, "hyperframe.manifest.json"));
cpSync(iifeSrc, join(stagingDir, "hyperframe.runtime.iife.js"));
console.log(`[build-zip] staged hyperframe.manifest.json + hyperframe.runtime.iife.js`);
}
// ELF header constants used by `assertLinuxX86_64Elf`. Header layout:
// bytes 0..3 magic `0x7F 'E' 'L' 'F'`, byte 4 EI_CLASS (`2` = ELFCLASS64),
// bytes 18..19 e_machine little-endian (`0x3E` = EM_X86_64).
const ELF_MAGIC = Buffer.from([0x7f, 0x45, 0x4c, 0x46]);
const ELF_CLASS_64 = 2;
const ELF_MACHINE_X86_64 = 0x3e;
const ELF_HEADER_BYTES = 20;
/**
* Verify the binary at `path` is a Linux x86-64 ELF executable, the only
* shape the Lambda runtime can exec. Throws with the canonical workaround
* (Docker `--platform=linux/amd64` or `npm_config_platform` /
* `npm_config_arch` overrides) so the build doesn't ship a silently
* broken zip when a non-Linux host's postinstall fetched the wrong arch.
*/
function assertLinuxX86_64Elf(path: string, label: string): void {
const head = readFileHead(path, ELF_HEADER_BYTES, label);
const isElf = head.subarray(0, 4).equals(ELF_MAGIC);
const isElf64 = head[4] === ELF_CLASS_64;
const machine = head.readUInt16LE(18);
if (isElf && isElf64 && machine === ELF_MACHINE_X86_64) return;
const magicHex = head.subarray(0, 4).toString("hex");
throw new Error(
`[build-zip] ${label} at ${path} is not a Linux x86-64 ELF executable ` +
`(magic=0x${magicHex}, ei_class=${head[4]}, e_machine=0x${machine.toString(16)}). ` +
`This usually means the deploy host's postinstall fetched a host-platform binary ` +
`(e.g. macOS arm64 ffmpeg) instead of the linux/x64 binary Lambda needs. ` +
`Re-run the build inside a linux/amd64 container, or pre-install with ` +
`\`npm_config_platform=linux npm_config_arch=x64\` so the package fetches the right binary.`,
);
}
function readFileHead(path: string, byteCount: number, label: string): Buffer {
const fd = openSync(path, "r");
try {
const buf = Buffer.alloc(byteCount);
const bytesRead = readSync(fd, buf, 0, byteCount, 0);
if (bytesRead < byteCount) {
throw new Error(
`[build-zip] ${label} at ${path} is too short — read ${bytesRead} of ${byteCount} bytes.`,
);
}
return buf;
} finally {
closeSync(fd);
}
}
function stageFfmpeg(stagingDir: string): void {
const binDir = join(stagingDir, "bin");
mkdirSync(binDir, { recursive: true });
// ffmpeg from `ffmpeg-static`. The package only ships the encoder
// binary; the audio pad/trim path also needs ffprobe, which comes
// from `ffprobe-static`.
//
// `ffmpeg-static`'s postinstall fetches a binary for the host platform
// (e.g. arm64 Mach-O on Apple Silicon macOS). Lambda runs Linux x86-64,
// so a build from a non-Linux host silently produces a zip that boots
// but fails at first ffmpeg invocation with `cannot execute binary file`.
// Verify the ELF header up front and bail with a clear message instead.
const ffmpegBinary = join(resolveModuleDir("ffmpeg-static"), "ffmpeg");
if (!existsSync(ffmpegBinary)) {
throw new Error(
`[build-zip] ffmpeg-static binary missing at ${ffmpegBinary}. Did postinstall run?`,
);
}
assertLinuxX86_64Elf(ffmpegBinary, "ffmpeg-static binary");
const ffmpegDest = join(binDir, "ffmpeg");
cpSync(ffmpegBinary, ffmpegDest);
chmodSync(ffmpegDest, 0o755);
// ffprobe lives at `ffprobe-static/bin/<platform>/<arch>/ffprobe`.
// The producer's `audioPadTrim` spawns `ffprobe` from PATH so we need
// it alongside ffmpeg under /var/task/bin/.
const ffprobeModule = resolveModuleDir("ffprobe-static");
const ffprobeCandidates = [
join(ffprobeModule, "bin", "linux", "x64", "ffprobe"),
join(ffprobeModule, "bin", "linux", "arm64", "ffprobe"),
];
const ffprobeBinary = ffprobeCandidates.find((p) => existsSync(p));
if (!ffprobeBinary) {
throw new Error(
`[build-zip] ffprobe-static binary not found under ${ffprobeModule}/bin/linux/. Did postinstall run?`,
);
}
const ffprobeDest = join(binDir, "ffprobe");
cpSync(ffprobeBinary, ffprobeDest);
chmodSync(ffprobeDest, 0o755);
console.log(`[build-zip] staged ffmpeg + ffprobe → bin/`);
}
function zipDirectory(sourceDir: string, zipPath: string): void {
const result = spawnSync("zip", ["-rq", zipPath, "."], { cwd: sourceDir, stdio: "inherit" });
if (result.status !== 0) {
throw new Error(`[build-zip] zip exited with status ${result.status}`);
}
}
function directorySizeBytes(dir: string): number {
// Use spawnSync (no shell) instead of execSync so `dir` is passed as
// an argv element rather than interpolated into a shell command —
// CodeQL's `js/shell-command-injected-from-environment` rule fires
// on the latter even with JSON-quoting. `du -sb` is Linux-only;
// build-zip is CI-side where Linux coreutils is present.
const result = spawnSync("du", ["-sb", dir], { encoding: "utf-8" });
if (result.status === 0 && result.stdout) {
const bytes = Number.parseInt(result.stdout.split(/\s+/)[0] ?? "0", 10);
if (!Number.isNaN(bytes)) return bytes;
}
return walkSize(dir);
}
function walkSize(dir: string): number {
let total = 0;
for (const entry of readdirSync(dir, { withFileTypes: true })) {
const full = join(dir, entry.name);
if (entry.isDirectory()) total += walkSize(full);
else if (entry.isFile()) total += statSync(full).size;
}
return total;
}
void main().catch((err) => {
console.error("[build-zip] failed:", err instanceof Error ? err.message : String(err));
if (err instanceof Error && err.stack) console.error(err.stack);
process.exit(1);
});