mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-10-30 09:16:55 +00:00
feat(aws-lambda): attach the arm64 video image to releases and pin it (#6070)
* feat(aws-lambda): attach the arm64 video image to releases and pin it The release builds the arm64 video function image from the handler ZIP it attaches, attaches it as an OCI layout tarball with its sha256, and pins it in prebuilt/video-image.json, read through prebuiltVideoImage(). A retried release reuses the image already attached. * fix(aws-lambda): keep the release pin script out of the published package pin-handler-zip.ts is run from a checkout by publish.yml; nothing in dist or the CLI imports it. Leaving it out keeps aws-lambda under its size budget with the video image reader added.
This commit is contained in:
Miguel Ángel
@@ -14,6 +14,11 @@ on:
|
||||
- "packages/aws-lambda/package.json"
|
||||
- "packages/aws-lambda/scripts/build-zip.ts"
|
||||
- ".github/workflows/lambda-arm64-image.yml"
|
||||
# On main it also seeds publish.yml's ffmpeg cache.
|
||||
push:
|
||||
branches: [main]
|
||||
paths:
|
||||
- "packages/aws-lambda/image/**"
|
||||
workflow_dispatch:
|
||||
|
||||
concurrency:
|
||||
@@ -35,6 +40,12 @@ jobs:
|
||||
docker buildx build --progress=plain --platform linux/arm64 --target artifact
|
||||
--output type=local,dest=out packages/aws-lambda/image
|
||||
- run: cat out/hf-media-check.txt && sha256sum out/*.tar.gz
|
||||
# Same path and key as publish.yml's lambda-arm64-ffmpeg job reads.
|
||||
- if: matrix.copy == 'a' && github.ref == 'refs/heads/main'
|
||||
uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
|
||||
with:
|
||||
path: out/
|
||||
key: hf-media-linux-aarch64-${{ hashFiles('packages/aws-lambda/image/**') }}
|
||||
- uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
|
||||
with:
|
||||
name: hf-media-${{ matrix.copy }}
|
||||
|
||||
@@ -13,10 +13,47 @@ on:
|
||||
branches: [main]
|
||||
|
||||
jobs:
|
||||
lambda-arm64-ffmpeg:
|
||||
name: Lambda arm64 ffmpeg
|
||||
runs-on: ubuntu-24.04-arm
|
||||
# A cache miss compiles ffmpeg natively, 15-20 minutes.
|
||||
timeout-minutes: 60
|
||||
permissions:
|
||||
contents: read
|
||||
env:
|
||||
EXPECTED_RELEASE_SHA: >-
|
||||
${{ github.event_name == 'pull_request'
|
||||
&& github.event.pull_request.merge_commit_sha
|
||||
|| github.sha }}
|
||||
if: >-
|
||||
github.event_name == 'push' ||
|
||||
(github.event.pull_request.merged == true &&
|
||||
startsWith(github.event.pull_request.head.ref, 'release/v'))
|
||||
steps:
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
with:
|
||||
ref: ${{ env.EXPECTED_RELEASE_SHA }}
|
||||
# lambda-arm64-image.yml saves this key on main; a release run's own saves reach only its re-runs.
|
||||
- id: cache
|
||||
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
|
||||
with:
|
||||
path: out/
|
||||
key: hf-media-linux-aarch64-${{ hashFiles('packages/aws-lambda/image/**') }}
|
||||
- if: steps.cache.outputs.cache-hit != 'true'
|
||||
run: >-
|
||||
docker buildx build --progress=plain --platform linux/arm64 --target artifact
|
||||
--output type=local,dest=out packages/aws-lambda/image
|
||||
- uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
|
||||
with:
|
||||
name: hf-media-linux-aarch64
|
||||
path: out/hf-media-linux-aarch64.tar.gz
|
||||
if-no-files-found: error
|
||||
|
||||
publish:
|
||||
name: Publish
|
||||
needs: lambda-arm64-ffmpeg
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20
|
||||
timeout-minutes: 30
|
||||
environment: npm-publish
|
||||
permissions:
|
||||
contents: write
|
||||
@@ -137,6 +174,33 @@ jobs:
|
||||
fi
|
||||
bun packages/aws-lambda/scripts/pin-handler-zip.ts
|
||||
|
||||
- uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
|
||||
with:
|
||||
name: hf-media-linux-aarch64
|
||||
path: dist/hf-media
|
||||
|
||||
# Built from the zip pinned above. The video stage only copies files, so no emulation is needed.
|
||||
- name: Build Lambda video image
|
||||
id: lambda_video_image
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
VERSION: ${{ steps.version.outputs.version }}
|
||||
REPO: ${{ github.repository }}
|
||||
run: |
|
||||
# A retried run reuses the image already on the release, so the pin still matches it.
|
||||
if gh release download "v${VERSION}" --repo "$REPO" --pattern hyperframes-lambda-video-arm64.oci.tar --dir dist; then
|
||||
echo "attached=true" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
bun packages/aws-lambda/scripts/stage-arm64-image.ts --zip=dist/hyperframes-lambda-handler.zip \
|
||||
--ffmpeg=dist/hf-media/hf-media-linux-aarch64.tar.gz --out=dist/arm64-image
|
||||
# OCI export needs a docker-container builder; no attestations, so index.json lists one image.
|
||||
docker buildx create --use
|
||||
docker buildx build --progress=plain --platform linux/arm64 --target video \
|
||||
--provenance=false --sbom=false --build-context task=dist/arm64-image/task \
|
||||
--output type=oci,dest=dist/hyperframes-lambda-video-arm64.oci.tar packages/aws-lambda/image
|
||||
fi
|
||||
bun packages/aws-lambda/scripts/pin-handler-zip.ts --video-image
|
||||
|
||||
- run: bun run verify:packed-manifests
|
||||
|
||||
- name: Create GitHub Release
|
||||
@@ -173,6 +237,14 @@ jobs:
|
||||
REPO: ${{ github.repository }}
|
||||
run: gh release upload "v${VERSION}" dist/hyperframes-lambda-handler.zip dist/hyperframes-lambda-handler.sha256 --repo "$REPO"
|
||||
|
||||
- name: Attach Lambda video image to release
|
||||
if: steps.lambda_video_image.outputs.attached != 'true'
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
VERSION: ${{ steps.version.outputs.version }}
|
||||
REPO: ${{ github.repository }}
|
||||
run: gh release upload "v${VERSION}" dist/hyperframes-lambda-video-arm64.oci.tar dist/hyperframes-lambda-video-arm64.sha256 --repo "$REPO"
|
||||
|
||||
- name: Publish packages
|
||||
env:
|
||||
NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
|
||||
|
||||
@@ -12,6 +12,7 @@
|
||||
"prebuilt/",
|
||||
"scripts/",
|
||||
"!scripts/*.test.ts",
|
||||
"!scripts/pin-handler-zip.ts",
|
||||
"README.md"
|
||||
],
|
||||
"type": "module",
|
||||
|
||||
@@ -3,8 +3,14 @@ import { createHash } from "node:crypto";
|
||||
import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { adapterPackageRoot, prebuiltHandler } from "../src/sdk/prebuiltHandler.js";
|
||||
import { pinHandlerZip } from "./pin-handler-zip.js";
|
||||
import {
|
||||
adapterPackageRoot,
|
||||
prebuiltHandler,
|
||||
prebuiltVideoImage,
|
||||
VIDEO_IMAGE_ASSET,
|
||||
VIDEO_IMAGE_MANIFEST_FILE,
|
||||
} from "../src/sdk/prebuiltHandler.js";
|
||||
import { pinHandlerZip, pinReleaseAsset } from "./pin-handler-zip.js";
|
||||
|
||||
const examples = join(import.meta.dir, "..", "..", "..", "examples", "aws-lambda");
|
||||
const realTemplate = readFileSync(join(examples, "template.yaml"), "utf-8");
|
||||
@@ -93,6 +99,31 @@ describe("pinHandlerZip", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("pinReleaseAsset", () => {
|
||||
it("pins the arm64 video image beside the handler zip without touching the zip's pin", () => {
|
||||
const { root, pkgDir } = fixtureRepo();
|
||||
writeFileSync(join(pkgDir, "dist", "handler.zip"), "zip");
|
||||
const zipPin = pinHandlerZip(root);
|
||||
const image = Buffer.from("oci layout tar");
|
||||
writeFileSync(join(root, "dist", VIDEO_IMAGE_ASSET), image);
|
||||
const sha256 = createHash("sha256").update(image).digest("hex");
|
||||
|
||||
const manifest = pinReleaseAsset(root, VIDEO_IMAGE_ASSET, VIDEO_IMAGE_MANIFEST_FILE);
|
||||
|
||||
expect(manifest).toEqual({
|
||||
version: "9.8.7",
|
||||
url: "https://github.com/heygen-com/hyperframes/releases/download/v9.8.7/hyperframes-lambda-video-arm64.oci.tar",
|
||||
sha256,
|
||||
bytes: image.length,
|
||||
});
|
||||
expect(readFileSync(join(root, "dist", "hyperframes-lambda-video-arm64.sha256"), "utf-8")).toBe(
|
||||
`${sha256} hyperframes-lambda-video-arm64.oci.tar\n`,
|
||||
);
|
||||
expect(prebuiltVideoImage(pkgDir)).toEqual(manifest);
|
||||
expect(prebuiltHandler(pkgDir)?.sha256).toBe(zipPin.sha256);
|
||||
});
|
||||
});
|
||||
|
||||
describe("prebuiltHandler", () => {
|
||||
it("reads from its own package root", () => {
|
||||
expect(adapterPackageRoot()).toBe(join(import.meta.dir, ".."));
|
||||
@@ -102,4 +133,11 @@ describe("prebuiltHandler", () => {
|
||||
const { pkgDir } = fixtureRepo();
|
||||
expect(prebuiltHandler(pkgDir)).toBeUndefined();
|
||||
});
|
||||
|
||||
it("has no video image when the package pins only the zip", () => {
|
||||
const { root, pkgDir } = fixtureRepo();
|
||||
writeFileSync(join(pkgDir, "dist", "handler.zip"), "zip");
|
||||
pinHandlerZip(root);
|
||||
expect(prebuiltVideoImage(pkgDir)).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
#!/usr/bin/env bun
|
||||
// Release step: writes the handler ZIP's pin (asset URL + sha256) and the SAM template into the
|
||||
// package. A retried release run reuses the ZIP already on the release.
|
||||
// package, or with --video-image the arm64 video image's pin. A retried release run reuses the
|
||||
// asset already on the release.
|
||||
import { createHash } from "node:crypto";
|
||||
import { existsSync, mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { dirname, join, resolve } from "node:path";
|
||||
@@ -8,17 +9,42 @@ import { fileURLToPath } from "node:url";
|
||||
import {
|
||||
HANDLER_MANIFEST_FILE,
|
||||
HANDLER_ZIP_ASSET,
|
||||
type HandlerZipManifest,
|
||||
PREBUILT_DIR,
|
||||
type ReleaseAssetManifest,
|
||||
VIDEO_IMAGE_ASSET,
|
||||
VIDEO_IMAGE_MANIFEST_FILE,
|
||||
} from "../src/sdk/prebuiltHandler.js";
|
||||
import { writePrebuiltTemplate } from "./prebuilt-template.mjs";
|
||||
|
||||
export function pinHandlerZip(repoRoot: string): HandlerZipManifest {
|
||||
/** Hashes `dist/<asset>`, writes its `.sha256` beside it and its pin to `prebuilt/<manifestFile>`. */
|
||||
export function pinReleaseAsset(
|
||||
repoRoot: string,
|
||||
asset: string,
|
||||
manifestFile: string,
|
||||
): ReleaseAssetManifest {
|
||||
const pkgDir = join(repoRoot, "packages", "aws-lambda");
|
||||
const pkg = JSON.parse(readFileSync(join(pkgDir, "package.json"), "utf-8")) as {
|
||||
version: string;
|
||||
repository: { url: string };
|
||||
};
|
||||
const bytes = readFileSync(join(repoRoot, "dist", asset));
|
||||
const sha256 = createHash("sha256").update(bytes).digest("hex");
|
||||
writeFileSync(join(repoRoot, "dist", `${asset.split(".")[0]}.sha256`), `${sha256} ${asset}\n`);
|
||||
|
||||
const manifest: ReleaseAssetManifest = {
|
||||
version: pkg.version,
|
||||
url: `${pkg.repository.url}/releases/download/v${pkg.version}/${asset}`,
|
||||
sha256,
|
||||
bytes: bytes.length,
|
||||
};
|
||||
const outDir = join(pkgDir, PREBUILT_DIR);
|
||||
mkdirSync(outDir, { recursive: true });
|
||||
writeFileSync(join(outDir, manifestFile), `${JSON.stringify(manifest, null, 2)}\n`);
|
||||
return manifest;
|
||||
}
|
||||
|
||||
export function pinHandlerZip(repoRoot: string): ReleaseAssetManifest {
|
||||
const pkgDir = join(repoRoot, "packages", "aws-lambda");
|
||||
const asset = join(repoRoot, "dist", HANDLER_ZIP_ASSET);
|
||||
const built = join(pkgDir, "dist", "handler.zip");
|
||||
if (existsSync(built)) {
|
||||
@@ -26,23 +52,14 @@ export function pinHandlerZip(repoRoot: string): HandlerZipManifest {
|
||||
renameSync(built, asset);
|
||||
rmSync(`${built}.manifest.json`, { force: true });
|
||||
}
|
||||
const bytes = readFileSync(asset);
|
||||
const sha256 = createHash("sha256").update(bytes).digest("hex");
|
||||
writeFileSync(asset.replace(/\.zip$/, ".sha256"), `${sha256} ${HANDLER_ZIP_ASSET}\n`);
|
||||
|
||||
const outDir = join(pkgDir, PREBUILT_DIR);
|
||||
writePrebuiltTemplate(repoRoot, outDir);
|
||||
const manifest: HandlerZipManifest = {
|
||||
version: pkg.version,
|
||||
url: `${pkg.repository.url}/releases/download/v${pkg.version}/${HANDLER_ZIP_ASSET}`,
|
||||
sha256,
|
||||
bytes: bytes.length,
|
||||
};
|
||||
writeFileSync(join(outDir, HANDLER_MANIFEST_FILE), `${JSON.stringify(manifest, null, 2)}\n`);
|
||||
return manifest;
|
||||
writePrebuiltTemplate(repoRoot, join(pkgDir, PREBUILT_DIR));
|
||||
return pinReleaseAsset(repoRoot, HANDLER_ZIP_ASSET, HANDLER_MANIFEST_FILE);
|
||||
}
|
||||
|
||||
if (import.meta.main) {
|
||||
const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), "../../..");
|
||||
console.log(JSON.stringify(pinHandlerZip(repoRoot), null, 2));
|
||||
const manifest = process.argv.includes("--video-image")
|
||||
? pinReleaseAsset(repoRoot, VIDEO_IMAGE_ASSET, VIDEO_IMAGE_MANIFEST_FILE)
|
||||
: pinHandlerZip(repoRoot);
|
||||
console.log(JSON.stringify(manifest, null, 2));
|
||||
}
|
||||
|
||||
@@ -39,6 +39,11 @@ export {
|
||||
validateStepFunctionsInputSize,
|
||||
validateVariablesPayload,
|
||||
} from "./validateConfig.js";
|
||||
export { type PrebuiltHandler, prebuiltHandler } from "./prebuiltHandler.js";
|
||||
export {
|
||||
type PrebuiltHandler,
|
||||
prebuiltHandler,
|
||||
prebuiltVideoImage,
|
||||
type ReleaseAssetManifest,
|
||||
} from "./prebuiltHandler.js";
|
||||
export type { SerializableDistributedRenderConfig } from "../events.js";
|
||||
export type { DistributedFormat } from "../formatExtension.js";
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// The handler ZIP pinned to this package version; the release workflow writes `prebuilt/`.
|
||||
// The release assets pinned to this package version; the release workflow writes `prebuilt/`.
|
||||
|
||||
import { existsSync, readFileSync } from "node:fs";
|
||||
import { dirname, join } from "node:path";
|
||||
@@ -7,15 +7,17 @@ import { fileURLToPath } from "node:url";
|
||||
export const HANDLER_ZIP_ASSET = "hyperframes-lambda-handler.zip";
|
||||
export const PREBUILT_DIR = "prebuilt";
|
||||
export const HANDLER_MANIFEST_FILE = "handler-zip.json";
|
||||
export const VIDEO_IMAGE_ASSET = "hyperframes-lambda-video-arm64.oci.tar";
|
||||
export const VIDEO_IMAGE_MANIFEST_FILE = "video-image.json";
|
||||
|
||||
export interface HandlerZipManifest {
|
||||
export interface ReleaseAssetManifest {
|
||||
version: string;
|
||||
url: string;
|
||||
sha256: string;
|
||||
bytes: number;
|
||||
}
|
||||
|
||||
export interface PrebuiltHandler extends HandlerZipManifest {
|
||||
export interface PrebuiltHandler extends ReleaseAssetManifest {
|
||||
/** SAM template whose `CodeUri` is `handler.zip` in the same directory. */
|
||||
templatePath: string;
|
||||
}
|
||||
@@ -34,11 +36,21 @@ export function adapterPackageRoot(): string {
|
||||
throw new Error(`[aws-lambda] no @hyperframes/aws-lambda package.json above ${from}`);
|
||||
}
|
||||
|
||||
function readManifest(root: string, file: string): ReleaseAssetManifest | undefined {
|
||||
const path = join(root, PREBUILT_DIR, file);
|
||||
if (!existsSync(path)) return undefined;
|
||||
return JSON.parse(readFileSync(path, "utf-8")) as ReleaseAssetManifest;
|
||||
}
|
||||
|
||||
/** The pinned handler for this install, or undefined when this build carries none. */
|
||||
export function prebuiltHandler(root: string = adapterPackageRoot()): PrebuiltHandler | undefined {
|
||||
const dir = join(root, PREBUILT_DIR);
|
||||
const manifestPath = join(dir, HANDLER_MANIFEST_FILE);
|
||||
if (!existsSync(manifestPath)) return undefined;
|
||||
const manifest = JSON.parse(readFileSync(manifestPath, "utf-8")) as HandlerZipManifest;
|
||||
return { ...manifest, templatePath: join(dir, "template.yaml") };
|
||||
const manifest = readManifest(root, HANDLER_MANIFEST_FILE);
|
||||
return manifest && { ...manifest, templatePath: join(root, PREBUILT_DIR, "template.yaml") };
|
||||
}
|
||||
|
||||
/** The pinned arm64 video function image (an OCI layout tarball), or undefined when none is pinned. */
|
||||
export function prebuiltVideoImage(
|
||||
root: string = adapterPackageRoot(),
|
||||
): ReleaseAssetManifest | undefined {
|
||||
return readManifest(root, VIDEO_IMAGE_MANIFEST_FILE);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user