mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-10-30 09:16:55 +00:00
feat(aws-lambda): arm64 video function image from the handler ZIP (#6064)
* feat(aws-lambda): arm64 video function image from the handler ZIP The image is the release handler ZIP with its x86 ffmpeg and Chrome swapped for the native ffmpeg tree and the sha256-pinned Sparticuz arm64 Chrome pack, so the x86 and arm64 functions run the same handler code. The image stage only copies, so any host builds it. CI builds it and runs ffmpeg, a real Chrome render and the Lambda runtime on an ARM runner. * fix(aws-lambda): launch Chrome in the image smoke the way Lambda does @sparticuz/chromium inflates its AL2023 libraries only when AWS_EXECUTION_ENV names a Node runtime; Lambda sets it, a bare docker run does not. The workflow also runs on @sparticuz/chromium bumps and handler ZIP layout changes, which re-stage the image. * fix(aws-lambda): smoke the arm64 image through its own Lambda runtime The smoke is now a handler invoked through the image's entrypoint, whose bootstrap sets AWS_EXECUTION_ENV the way Lambda does, so @sparticuz/chromium takes its Lambda branch with nothing set by hand. It returns the value it saw. * fix(aws-lambda): agree the move of the ffmpeg build workflow into the image workflow * fix(aws-lambda): build the Chrome pack URL from the pin and hand the checked pack to tar in memory
This commit is contained in:
Miguel Ángel
@@ -0,0 +1,101 @@
|
||||
import { afterEach, describe, expect, it } from "bun:test";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { createHash } from "node:crypto";
|
||||
import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { dirname, join } from "node:path";
|
||||
import { CHROMIUM_ARM64_PACK_SHA256, stageArm64Image } from "./stage-arm64-image.js";
|
||||
|
||||
const X86_64 = 0x3e;
|
||||
const AARCH64 = 0xb7;
|
||||
const dirs: string[] = [];
|
||||
|
||||
afterEach(() => {
|
||||
for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
function elf(machine: number): Buffer {
|
||||
const head = Buffer.alloc(64);
|
||||
head.writeUInt32BE(0x7f454c46, 0);
|
||||
head[4] = 2;
|
||||
head.writeUInt16LE(machine, 18);
|
||||
return head;
|
||||
}
|
||||
|
||||
function tree(files: Record<string, string | Buffer>): string {
|
||||
const dir = mkdtempSync(join(tmpdir(), "stage-arm64-"));
|
||||
dirs.push(dir);
|
||||
for (const [path, body] of Object.entries(files)) {
|
||||
mkdirSync(dirname(join(dir, path)), { recursive: true });
|
||||
writeFileSync(join(dir, path), body);
|
||||
}
|
||||
return dir;
|
||||
}
|
||||
|
||||
function pack(cmd: string, args: string[], cwd: string): void {
|
||||
expect(spawnSync(cmd, args, { cwd }).status).toBe(0);
|
||||
}
|
||||
|
||||
function fixture(opts: { ffmpegMachine?: number; zipExtra?: Record<string, string> } = {}) {
|
||||
const zipDir = tree({
|
||||
"handler.mjs": "export const handler = () => {};",
|
||||
"bin/ffmpeg": elf(X86_64),
|
||||
"bin/ffprobe": elf(X86_64),
|
||||
"node_modules/@sparticuz/chromium/package.json": JSON.stringify({ version: "148.0.0" }),
|
||||
"node_modules/@sparticuz/chromium/bin/chromium.br": "x64 chrome",
|
||||
...opts.zipExtra,
|
||||
});
|
||||
pack("zip", ["-qr", "handler.zip", "."], zipDir);
|
||||
const machine = opts.ffmpegMachine ?? AARCH64;
|
||||
const mediaDir = tree({
|
||||
"hf-media/bin/ffmpeg": elf(machine),
|
||||
"hf-media/bin/ffprobe": elf(machine),
|
||||
"hf-media/lib/libgnutls.so.30": "gnutls",
|
||||
});
|
||||
pack("tar", ["-czf", "hf-media.tar.gz", "hf-media"], mediaDir);
|
||||
const packDir = tree({ "chromium.br": "arm64 chrome", "al2023.tar.br": "arm64 libs" });
|
||||
pack("tar", ["-cf", "pack.tar", "chromium.br", "al2023.tar.br"], packDir);
|
||||
const packBytes = readFileSync(join(packDir, "pack.tar"));
|
||||
return {
|
||||
handlerZip: join(zipDir, "handler.zip"),
|
||||
ffmpegTarball: join(mediaDir, "hf-media.tar.gz"),
|
||||
outDir: join(tree({}), "out"),
|
||||
fetchPack: async () => packBytes,
|
||||
packPins: { "148.0.0": createHash("sha256").update(packBytes).digest("hex") },
|
||||
};
|
||||
}
|
||||
|
||||
describe("stageArm64Image", () => {
|
||||
it("swaps the handler ZIP's x86 ffmpeg and Chrome for the arm64 ones", async () => {
|
||||
const task = await stageArm64Image(fixture());
|
||||
expect(readFileSync(join(task, "bin/ffmpeg")).readUInt16LE(18)).toBe(AARCH64);
|
||||
expect(readFileSync(join(task, "bin/ffprobe")).readUInt16LE(18)).toBe(AARCH64);
|
||||
expect(readFileSync(join(task, "lib/libgnutls.so.30"), "utf-8")).toBe("gnutls");
|
||||
const chromeBin = join(task, "node_modules/@sparticuz/chromium/bin");
|
||||
expect(readFileSync(join(chromeBin, "chromium.br"), "utf-8")).toBe("arm64 chrome");
|
||||
expect(readFileSync(join(chromeBin, "al2023.tar.br"), "utf-8")).toBe("arm64 libs");
|
||||
expect(readFileSync(join(task, "handler.mjs"), "utf-8")).toContain("handler");
|
||||
});
|
||||
|
||||
it("refuses an ffmpeg tarball that is not arm64", async () => {
|
||||
await expect(stageArm64Image(fixture({ ffmpegMachine: X86_64 }))).rejects.toThrow(
|
||||
"is not a 64-bit arm64 ELF binary",
|
||||
);
|
||||
});
|
||||
|
||||
it("refuses a Chrome pack whose sha256 is not the pinned one", async () => {
|
||||
const opts = { ...fixture(), packPins: CHROMIUM_ARM64_PACK_SHA256 };
|
||||
await expect(stageArm64Image(opts)).rejects.toThrow("pinned c18a5241");
|
||||
});
|
||||
|
||||
it("refuses a @sparticuz/chromium version with no pinned arm64 pack", async () => {
|
||||
await expect(stageArm64Image({ ...fixture(), packPins: {} })).rejects.toThrow(
|
||||
"no arm64 Chrome pack pinned for @sparticuz/chromium 148.0.0",
|
||||
);
|
||||
});
|
||||
|
||||
it("refuses a handler ZIP built with chrome-headless-shell", async () => {
|
||||
const opts = fixture({ zipExtra: { "bin/chrome-headless-shell": "x86 chrome" } });
|
||||
await expect(stageArm64Image(opts)).rejects.toThrow("sparticuz Chrome only");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,114 @@
|
||||
#!/usr/bin/env bun
|
||||
// Stages /var/task for the arm64 video function image: the handler ZIP with its x86 ffmpeg and Chrome
|
||||
// swapped for the native ffmpeg tree (image/Dockerfile's artifact) and Sparticuz's arm64 Chrome pack.
|
||||
// bun packages/aws-lambda/scripts/stage-arm64-image.ts --ffmpeg=<hf-media tarball> [--zip=..] [--out=..]
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { createHash } from "node:crypto";
|
||||
import {
|
||||
closeSync,
|
||||
existsSync,
|
||||
mkdirSync,
|
||||
openSync,
|
||||
readFileSync,
|
||||
readSync,
|
||||
rmSync,
|
||||
} from "node:fs";
|
||||
import { dirname, join, resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
// The npm package ships x64 Chrome only. Bumping @sparticuz/chromium fails here until its arm64 pack is pinned.
|
||||
export const CHROMIUM_ARM64_PACK_SHA256: Record<string, string> = {
|
||||
"148.0.0": "c18a5241b0a460cba233fff3abccd6678ebdcf855b8548bb7c58240a477f5f0e",
|
||||
};
|
||||
|
||||
export interface StageOptions {
|
||||
handlerZip: string;
|
||||
ffmpegTarball: string;
|
||||
outDir: string;
|
||||
fetchPack?: (url: string) => Promise<Uint8Array>;
|
||||
packPins?: Record<string, string>;
|
||||
}
|
||||
|
||||
async function fetchBytes(url: string): Promise<Uint8Array> {
|
||||
const res = await fetch(url);
|
||||
if (!res.ok) throw new Error(`[stage-arm64-image] GET ${url} answered ${res.status}`);
|
||||
return new Uint8Array(await res.arrayBuffer());
|
||||
}
|
||||
|
||||
function run(cmd: string, args: string[], input?: Uint8Array): void {
|
||||
const result = spawnSync(cmd, args, {
|
||||
stdio: [input ? "pipe" : "inherit", "inherit", "inherit"],
|
||||
input,
|
||||
});
|
||||
if (result.status !== 0) {
|
||||
throw new Error(`[stage-arm64-image] ${cmd} ${args.join(" ")} exited ${result.status}`);
|
||||
}
|
||||
}
|
||||
|
||||
export function assertArm64Elf(path: string): void {
|
||||
const head = Buffer.alloc(20);
|
||||
const fd = openSync(path, "r");
|
||||
try {
|
||||
readSync(fd, head, 0, head.length, 0);
|
||||
} finally {
|
||||
closeSync(fd);
|
||||
}
|
||||
const elf64 = head.readUInt32BE(0) === 0x7f454c46 && head[4] === 2;
|
||||
if (!elf64 || head.readUInt16LE(18) !== 0xb7) {
|
||||
throw new Error(`[stage-arm64-image] ${path} is not a 64-bit arm64 ELF binary`);
|
||||
}
|
||||
}
|
||||
|
||||
async function stageChromiumArm64(chromiumDir: string, opts: StageOptions) {
|
||||
const { version } = JSON.parse(readFileSync(join(chromiumDir, "package.json"), "utf-8")) as {
|
||||
version: string;
|
||||
};
|
||||
// The URL is built from the pin table's own key, never from the version read off disk.
|
||||
const [pinnedVersion, pinned] =
|
||||
Object.entries(opts.packPins ?? CHROMIUM_ARM64_PACK_SHA256).find(([v]) => v === version) ?? [];
|
||||
if (!pinnedVersion || !pinned) {
|
||||
throw new Error(
|
||||
`[stage-arm64-image] no arm64 Chrome pack pinned for @sparticuz/chromium ${version}; ` +
|
||||
"add its sha256 to CHROMIUM_ARM64_PACK_SHA256",
|
||||
);
|
||||
}
|
||||
const url = `https://github.com/Sparticuz/chromium/releases/download/v${pinnedVersion}/chromium-v${pinnedVersion}-pack.arm64.tar`;
|
||||
const pack = await (opts.fetchPack ?? fetchBytes)(url);
|
||||
const actual = createHash("sha256").update(pack).digest("hex");
|
||||
if (actual !== pinned) {
|
||||
throw new Error(`[stage-arm64-image] ${url} has sha256 ${actual}, pinned ${pinned}`);
|
||||
}
|
||||
// Only bytes that matched the pin reach tar, over stdin; the pack is never written to a file.
|
||||
run("tar", ["-xf", "-", "-C", join(chromiumDir, "bin")], pack);
|
||||
}
|
||||
|
||||
export async function stageArm64Image(opts: StageOptions): Promise<string> {
|
||||
const task = join(opts.outDir, "task");
|
||||
rmSync(opts.outDir, { recursive: true, force: true });
|
||||
mkdirSync(task, { recursive: true });
|
||||
run("unzip", ["-q", opts.handlerZip, "-d", task]);
|
||||
if (existsSync(join(task, "bin", "chrome-headless-shell"))) {
|
||||
throw new Error("[stage-arm64-image] the arm64 image bundles sparticuz Chrome only");
|
||||
}
|
||||
rmSync(join(task, "bin", "ffmpeg"));
|
||||
rmSync(join(task, "bin", "ffprobe"));
|
||||
run("tar", ["-xzf", opts.ffmpegTarball, "-C", task, "--strip-components=1"]);
|
||||
for (const bin of ["ffmpeg", "ffprobe"]) assertArm64Elf(join(task, "bin", bin));
|
||||
await stageChromiumArm64(join(task, "node_modules", "@sparticuz", "chromium"), opts);
|
||||
return task;
|
||||
}
|
||||
|
||||
if (import.meta.main) {
|
||||
const distDir = resolve(dirname(fileURLToPath(import.meta.url)), "..", "dist");
|
||||
const flag = (name: string) =>
|
||||
process.argv.find((arg) => arg.startsWith(`--${name}=`))?.slice(name.length + 3);
|
||||
const ffmpegTarball = flag("ffmpeg");
|
||||
if (!ffmpegTarball)
|
||||
throw new Error("[stage-arm64-image] --ffmpeg=<hf-media tarball> is required");
|
||||
const task = await stageArm64Image({
|
||||
handlerZip: flag("zip") ?? join(distDir, "handler.zip"),
|
||||
ffmpegTarball,
|
||||
outDir: flag("out") ?? join(distDir, "arm64-image"),
|
||||
});
|
||||
console.log(`[stage-arm64-image] staged ${task}`);
|
||||
}
|
||||
Reference in New Issue
Block a user