From 8d562a3d918b215f145d84507000ff7a67b68352 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81ngel?= Date: Fri, 23 Oct 2026 03:52:35 +0000 Subject: [PATCH] feat(aws-lambda): attach the arm64 video image to releases and pin it (#6070) * feat(aws-lambda): attach the arm64 video image to releases and pin it The release builds the arm64 video function image from the handler ZIP it attaches, attaches it as an OCI layout tarball with its sha256, and pins it in prebuilt/video-image.json, read through prebuiltVideoImage(). A retried release reuses the image already attached. * fix(aws-lambda): keep the release pin script out of the published package pin-handler-zip.ts is run from a checkout by publish.yml; nothing in dist or the CLI imports it. Leaving it out keeps aws-lambda under its size budget with the video image reader added. --- .github/workflows/lambda-arm64-image.yml | 11 +++ .github/workflows/publish.yml | 74 ++++++++++++++++++- packages/aws-lambda/package.json | 1 + .../scripts/pin-handler-zip.test.ts | 42 ++++++++++- .../aws-lambda/scripts/pin-handler-zip.ts | 53 ++++++++----- packages/aws-lambda/src/sdk/index.ts | 7 +- .../aws-lambda/src/sdk/prebuiltHandler.ts | 28 +++++-- 7 files changed, 186 insertions(+), 30 deletions(-) diff --git a/.github/workflows/lambda-arm64-image.yml b/.github/workflows/lambda-arm64-image.yml index 1f330694b2..8ae3b2365b 100644 --- a/.github/workflows/lambda-arm64-image.yml +++ b/.github/workflows/lambda-arm64-image.yml @@ -14,6 +14,11 @@ on: - "packages/aws-lambda/package.json" - "packages/aws-lambda/scripts/build-zip.ts" - ".github/workflows/lambda-arm64-image.yml" + # On main it also seeds publish.yml's ffmpeg cache. + push: + branches: [main] + paths: + - "packages/aws-lambda/image/**" workflow_dispatch: concurrency: @@ -35,6 +40,12 @@ jobs: docker buildx build --progress=plain --platform linux/arm64 --target artifact --output type=local,dest=out packages/aws-lambda/image - run: cat out/hf-media-check.txt && sha256sum out/*.tar.gz + # Same path and key as publish.yml's lambda-arm64-ffmpeg job reads. + - if: matrix.copy == 'a' && github.ref == 'refs/heads/main' + uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 + with: + path: out/ + key: hf-media-linux-aarch64-${{ hashFiles('packages/aws-lambda/image/**') }} - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 with: name: hf-media-${{ matrix.copy }} diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index e3dbef36ca..7376b59886 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -13,10 +13,47 @@ on: branches: [main] jobs: + lambda-arm64-ffmpeg: + name: Lambda arm64 ffmpeg + runs-on: ubuntu-24.04-arm + # A cache miss compiles ffmpeg natively, 15-20 minutes. + timeout-minutes: 60 + permissions: + contents: read + env: + EXPECTED_RELEASE_SHA: >- + ${{ github.event_name == 'pull_request' + && github.event.pull_request.merge_commit_sha + || github.sha }} + if: >- + github.event_name == 'push' || + (github.event.pull_request.merged == true && + startsWith(github.event.pull_request.head.ref, 'release/v')) + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + ref: ${{ env.EXPECTED_RELEASE_SHA }} + # lambda-arm64-image.yml saves this key on main; a release run's own saves reach only its re-runs. + - id: cache + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 + with: + path: out/ + key: hf-media-linux-aarch64-${{ hashFiles('packages/aws-lambda/image/**') }} + - if: steps.cache.outputs.cache-hit != 'true' + run: >- + docker buildx build --progress=plain --platform linux/arm64 --target artifact + --output type=local,dest=out packages/aws-lambda/image + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: hf-media-linux-aarch64 + path: out/hf-media-linux-aarch64.tar.gz + if-no-files-found: error + publish: name: Publish + needs: lambda-arm64-ffmpeg runs-on: ubuntu-latest - timeout-minutes: 20 + timeout-minutes: 30 environment: npm-publish permissions: contents: write @@ -137,6 +174,33 @@ jobs: fi bun packages/aws-lambda/scripts/pin-handler-zip.ts + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + name: hf-media-linux-aarch64 + path: dist/hf-media + + # Built from the zip pinned above. The video stage only copies files, so no emulation is needed. + - name: Build Lambda video image + id: lambda_video_image + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + VERSION: ${{ steps.version.outputs.version }} + REPO: ${{ github.repository }} + run: | + # A retried run reuses the image already on the release, so the pin still matches it. + if gh release download "v${VERSION}" --repo "$REPO" --pattern hyperframes-lambda-video-arm64.oci.tar --dir dist; then + echo "attached=true" >> "$GITHUB_OUTPUT" + else + bun packages/aws-lambda/scripts/stage-arm64-image.ts --zip=dist/hyperframes-lambda-handler.zip \ + --ffmpeg=dist/hf-media/hf-media-linux-aarch64.tar.gz --out=dist/arm64-image + # OCI export needs a docker-container builder; no attestations, so index.json lists one image. + docker buildx create --use + docker buildx build --progress=plain --platform linux/arm64 --target video \ + --provenance=false --sbom=false --build-context task=dist/arm64-image/task \ + --output type=oci,dest=dist/hyperframes-lambda-video-arm64.oci.tar packages/aws-lambda/image + fi + bun packages/aws-lambda/scripts/pin-handler-zip.ts --video-image + - run: bun run verify:packed-manifests - name: Create GitHub Release @@ -173,6 +237,14 @@ jobs: REPO: ${{ github.repository }} run: gh release upload "v${VERSION}" dist/hyperframes-lambda-handler.zip dist/hyperframes-lambda-handler.sha256 --repo "$REPO" + - name: Attach Lambda video image to release + if: steps.lambda_video_image.outputs.attached != 'true' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + VERSION: ${{ steps.version.outputs.version }} + REPO: ${{ github.repository }} + run: gh release upload "v${VERSION}" dist/hyperframes-lambda-video-arm64.oci.tar dist/hyperframes-lambda-video-arm64.sha256 --repo "$REPO" + - name: Publish packages env: NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} diff --git a/packages/aws-lambda/package.json b/packages/aws-lambda/package.json index 8ac6589d38..0c6eaf7f4f 100644 --- a/packages/aws-lambda/package.json +++ b/packages/aws-lambda/package.json @@ -12,6 +12,7 @@ "prebuilt/", "scripts/", "!scripts/*.test.ts", + "!scripts/pin-handler-zip.ts", "README.md" ], "type": "module", diff --git a/packages/aws-lambda/scripts/pin-handler-zip.test.ts b/packages/aws-lambda/scripts/pin-handler-zip.test.ts index a57573d47e..52c299c5b2 100644 --- a/packages/aws-lambda/scripts/pin-handler-zip.test.ts +++ b/packages/aws-lambda/scripts/pin-handler-zip.test.ts @@ -3,8 +3,14 @@ import { createHash } from "node:crypto"; import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { adapterPackageRoot, prebuiltHandler } from "../src/sdk/prebuiltHandler.js"; -import { pinHandlerZip } from "./pin-handler-zip.js"; +import { + adapterPackageRoot, + prebuiltHandler, + prebuiltVideoImage, + VIDEO_IMAGE_ASSET, + VIDEO_IMAGE_MANIFEST_FILE, +} from "../src/sdk/prebuiltHandler.js"; +import { pinHandlerZip, pinReleaseAsset } from "./pin-handler-zip.js"; const examples = join(import.meta.dir, "..", "..", "..", "examples", "aws-lambda"); const realTemplate = readFileSync(join(examples, "template.yaml"), "utf-8"); @@ -93,6 +99,31 @@ describe("pinHandlerZip", () => { }); }); +describe("pinReleaseAsset", () => { + it("pins the arm64 video image beside the handler zip without touching the zip's pin", () => { + const { root, pkgDir } = fixtureRepo(); + writeFileSync(join(pkgDir, "dist", "handler.zip"), "zip"); + const zipPin = pinHandlerZip(root); + const image = Buffer.from("oci layout tar"); + writeFileSync(join(root, "dist", VIDEO_IMAGE_ASSET), image); + const sha256 = createHash("sha256").update(image).digest("hex"); + + const manifest = pinReleaseAsset(root, VIDEO_IMAGE_ASSET, VIDEO_IMAGE_MANIFEST_FILE); + + expect(manifest).toEqual({ + version: "9.8.7", + url: "https://github.com/heygen-com/hyperframes/releases/download/v9.8.7/hyperframes-lambda-video-arm64.oci.tar", + sha256, + bytes: image.length, + }); + expect(readFileSync(join(root, "dist", "hyperframes-lambda-video-arm64.sha256"), "utf-8")).toBe( + `${sha256} hyperframes-lambda-video-arm64.oci.tar\n`, + ); + expect(prebuiltVideoImage(pkgDir)).toEqual(manifest); + expect(prebuiltHandler(pkgDir)?.sha256).toBe(zipPin.sha256); + }); +}); + describe("prebuiltHandler", () => { it("reads from its own package root", () => { expect(adapterPackageRoot()).toBe(join(import.meta.dir, "..")); @@ -102,4 +133,11 @@ describe("prebuiltHandler", () => { const { pkgDir } = fixtureRepo(); expect(prebuiltHandler(pkgDir)).toBeUndefined(); }); + + it("has no video image when the package pins only the zip", () => { + const { root, pkgDir } = fixtureRepo(); + writeFileSync(join(pkgDir, "dist", "handler.zip"), "zip"); + pinHandlerZip(root); + expect(prebuiltVideoImage(pkgDir)).toBeUndefined(); + }); }); diff --git a/packages/aws-lambda/scripts/pin-handler-zip.ts b/packages/aws-lambda/scripts/pin-handler-zip.ts index 60f8c607e7..361b8a3e07 100644 --- a/packages/aws-lambda/scripts/pin-handler-zip.ts +++ b/packages/aws-lambda/scripts/pin-handler-zip.ts @@ -1,6 +1,7 @@ #!/usr/bin/env bun // Release step: writes the handler ZIP's pin (asset URL + sha256) and the SAM template into the -// package. A retried release run reuses the ZIP already on the release. +// package, or with --video-image the arm64 video image's pin. A retried release run reuses the +// asset already on the release. import { createHash } from "node:crypto"; import { existsSync, mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs"; import { dirname, join, resolve } from "node:path"; @@ -8,17 +9,42 @@ import { fileURLToPath } from "node:url"; import { HANDLER_MANIFEST_FILE, HANDLER_ZIP_ASSET, - type HandlerZipManifest, PREBUILT_DIR, + type ReleaseAssetManifest, + VIDEO_IMAGE_ASSET, + VIDEO_IMAGE_MANIFEST_FILE, } from "../src/sdk/prebuiltHandler.js"; import { writePrebuiltTemplate } from "./prebuilt-template.mjs"; -export function pinHandlerZip(repoRoot: string): HandlerZipManifest { +/** Hashes `dist/`, writes its `.sha256` beside it and its pin to `prebuilt/`. */ +export function pinReleaseAsset( + repoRoot: string, + asset: string, + manifestFile: string, +): ReleaseAssetManifest { const pkgDir = join(repoRoot, "packages", "aws-lambda"); const pkg = JSON.parse(readFileSync(join(pkgDir, "package.json"), "utf-8")) as { version: string; repository: { url: string }; }; + const bytes = readFileSync(join(repoRoot, "dist", asset)); + const sha256 = createHash("sha256").update(bytes).digest("hex"); + writeFileSync(join(repoRoot, "dist", `${asset.split(".")[0]}.sha256`), `${sha256} ${asset}\n`); + + const manifest: ReleaseAssetManifest = { + version: pkg.version, + url: `${pkg.repository.url}/releases/download/v${pkg.version}/${asset}`, + sha256, + bytes: bytes.length, + }; + const outDir = join(pkgDir, PREBUILT_DIR); + mkdirSync(outDir, { recursive: true }); + writeFileSync(join(outDir, manifestFile), `${JSON.stringify(manifest, null, 2)}\n`); + return manifest; +} + +export function pinHandlerZip(repoRoot: string): ReleaseAssetManifest { + const pkgDir = join(repoRoot, "packages", "aws-lambda"); const asset = join(repoRoot, "dist", HANDLER_ZIP_ASSET); const built = join(pkgDir, "dist", "handler.zip"); if (existsSync(built)) { @@ -26,23 +52,14 @@ export function pinHandlerZip(repoRoot: string): HandlerZipManifest { renameSync(built, asset); rmSync(`${built}.manifest.json`, { force: true }); } - const bytes = readFileSync(asset); - const sha256 = createHash("sha256").update(bytes).digest("hex"); - writeFileSync(asset.replace(/\.zip$/, ".sha256"), `${sha256} ${HANDLER_ZIP_ASSET}\n`); - - const outDir = join(pkgDir, PREBUILT_DIR); - writePrebuiltTemplate(repoRoot, outDir); - const manifest: HandlerZipManifest = { - version: pkg.version, - url: `${pkg.repository.url}/releases/download/v${pkg.version}/${HANDLER_ZIP_ASSET}`, - sha256, - bytes: bytes.length, - }; - writeFileSync(join(outDir, HANDLER_MANIFEST_FILE), `${JSON.stringify(manifest, null, 2)}\n`); - return manifest; + writePrebuiltTemplate(repoRoot, join(pkgDir, PREBUILT_DIR)); + return pinReleaseAsset(repoRoot, HANDLER_ZIP_ASSET, HANDLER_MANIFEST_FILE); } if (import.meta.main) { const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), "../../.."); - console.log(JSON.stringify(pinHandlerZip(repoRoot), null, 2)); + const manifest = process.argv.includes("--video-image") + ? pinReleaseAsset(repoRoot, VIDEO_IMAGE_ASSET, VIDEO_IMAGE_MANIFEST_FILE) + : pinHandlerZip(repoRoot); + console.log(JSON.stringify(manifest, null, 2)); } diff --git a/packages/aws-lambda/src/sdk/index.ts b/packages/aws-lambda/src/sdk/index.ts index c03f7bbfbd..7fe1460600 100644 --- a/packages/aws-lambda/src/sdk/index.ts +++ b/packages/aws-lambda/src/sdk/index.ts @@ -39,6 +39,11 @@ export { validateStepFunctionsInputSize, validateVariablesPayload, } from "./validateConfig.js"; -export { type PrebuiltHandler, prebuiltHandler } from "./prebuiltHandler.js"; +export { + type PrebuiltHandler, + prebuiltHandler, + prebuiltVideoImage, + type ReleaseAssetManifest, +} from "./prebuiltHandler.js"; export type { SerializableDistributedRenderConfig } from "../events.js"; export type { DistributedFormat } from "../formatExtension.js"; diff --git a/packages/aws-lambda/src/sdk/prebuiltHandler.ts b/packages/aws-lambda/src/sdk/prebuiltHandler.ts index 6808530503..9dddb50d9c 100644 --- a/packages/aws-lambda/src/sdk/prebuiltHandler.ts +++ b/packages/aws-lambda/src/sdk/prebuiltHandler.ts @@ -1,4 +1,4 @@ -// The handler ZIP pinned to this package version; the release workflow writes `prebuilt/`. +// The release assets pinned to this package version; the release workflow writes `prebuilt/`. import { existsSync, readFileSync } from "node:fs"; import { dirname, join } from "node:path"; @@ -7,15 +7,17 @@ import { fileURLToPath } from "node:url"; export const HANDLER_ZIP_ASSET = "hyperframes-lambda-handler.zip"; export const PREBUILT_DIR = "prebuilt"; export const HANDLER_MANIFEST_FILE = "handler-zip.json"; +export const VIDEO_IMAGE_ASSET = "hyperframes-lambda-video-arm64.oci.tar"; +export const VIDEO_IMAGE_MANIFEST_FILE = "video-image.json"; -export interface HandlerZipManifest { +export interface ReleaseAssetManifest { version: string; url: string; sha256: string; bytes: number; } -export interface PrebuiltHandler extends HandlerZipManifest { +export interface PrebuiltHandler extends ReleaseAssetManifest { /** SAM template whose `CodeUri` is `handler.zip` in the same directory. */ templatePath: string; } @@ -34,11 +36,21 @@ export function adapterPackageRoot(): string { throw new Error(`[aws-lambda] no @hyperframes/aws-lambda package.json above ${from}`); } +function readManifest(root: string, file: string): ReleaseAssetManifest | undefined { + const path = join(root, PREBUILT_DIR, file); + if (!existsSync(path)) return undefined; + return JSON.parse(readFileSync(path, "utf-8")) as ReleaseAssetManifest; +} + /** The pinned handler for this install, or undefined when this build carries none. */ export function prebuiltHandler(root: string = adapterPackageRoot()): PrebuiltHandler | undefined { - const dir = join(root, PREBUILT_DIR); - const manifestPath = join(dir, HANDLER_MANIFEST_FILE); - if (!existsSync(manifestPath)) return undefined; - const manifest = JSON.parse(readFileSync(manifestPath, "utf-8")) as HandlerZipManifest; - return { ...manifest, templatePath: join(dir, "template.yaml") }; + const manifest = readManifest(root, HANDLER_MANIFEST_FILE); + return manifest && { ...manifest, templatePath: join(root, PREBUILT_DIR, "template.yaml") }; +} + +/** The pinned arm64 video function image (an OCI layout tarball), or undefined when none is pinned. */ +export function prebuiltVideoImage( + root: string = adapterPackageRoot(), +): ReleaseAssetManifest | undefined { + return readManifest(root, VIDEO_IMAGE_MANIFEST_FILE); }