mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-10-30 09:16:55 +00:00
build(aws-lambda): pinned native ffmpeg build for an arm64 Lambda image (#6062)
* build(aws-lambda): pinned native ffmpeg build for an arm64 Lambda image One recipe, run per architecture on the Lambda Node 22 base it ships on. Codec libraries link statically; glibc and GnuTLS stay dynamic, with the two libraries the bare base lacks bundled beside the binaries. A check stage on the bare base proves every encoder, decoder, filter and protocol the engine calls, and the artifact cannot be produced without it. CI builds it on two ARM runners and requires byte-identical tarballs. * fix(aws-lambda): link GnuTLS dynamically in the arm64 ffmpeg build With --static pkg-config flags, gnutls.pc names a libgmp.a from its own RPM build tree and configure rejected GnuTLS. The codec libraries exist only as static archives, so they still link statically. The recipe moves to image/, which will also hold the video function image stage. * fix(aws-lambda): build x265 for arm64 with the base image compiler x265 builds its SVE2 kernels with -march=armv9-a, which gcc 11 rejects. Turn them off, and stop a failed x265 build from reaching ffmpeg configure as a misleading "x265 not found". * fix(aws-lambda): pin x264 to the revision the x86 ffmpeg uses The x86 ffmpeg-static 7.0.2 encodes with x264 r3191 (4613ac3); match it. * fix(aws-lambda): 10-bit x265 in the arm64 ffmpeg, checked by output format x265 now builds the 10-bit library and links it into the 8-bit one, so HDR renders get yuv420p10le instead of a silent 8-bit fallback. The check asserts the encoded pixel format rather than a successful command, and its ffprobe calls terminate options with -- like every other caller.
This commit is contained in:
Miguel Ángel
@@ -0,0 +1,47 @@
|
||||
name: Lambda arm64 ffmpeg
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
# Builds the native ffmpeg for the arm64 Lambda video function on two separate ARM runners.
|
||||
# Each build runs the codec check on the bare Lambda base; the tarballs must be byte-identical.
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- "packages/aws-lambda/image/**"
|
||||
- ".github/workflows/lambda-ffmpeg.yml"
|
||||
workflow_dispatch:
|
||||
|
||||
concurrency:
|
||||
group: lambda-ffmpeg-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
build:
|
||||
name: Build ${{ matrix.copy }}
|
||||
runs-on: ubuntu-24.04-arm
|
||||
timeout-minutes: 90
|
||||
strategy:
|
||||
matrix:
|
||||
copy: [a, b]
|
||||
steps:
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
- name: Build and check
|
||||
run: >-
|
||||
docker buildx build --progress=plain --platform linux/arm64 --target artifact
|
||||
--output type=local,dest=out packages/aws-lambda/image
|
||||
- run: cat out/hf-media-check.txt && sha256sum out/*.tar.gz
|
||||
- uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
|
||||
with:
|
||||
name: hf-media-${{ matrix.copy }}
|
||||
path: out/
|
||||
retention-days: 7
|
||||
|
||||
reproducible:
|
||||
name: Builds match
|
||||
needs: build
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
- uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
|
||||
- run: cmp hf-media-a/hf-media-linux-aarch64.tar.gz hf-media-b/hf-media-linux-aarch64.tar.gz
|
||||
@@ -0,0 +1,34 @@
|
||||
# The arm64 Lambda video function's native ffmpeg, built on the Lambda Node 22 base it runs on:
|
||||
# docker buildx build --platform linux/arm64 --target artifact --output type=local,dest=out packages/aws-lambda/image
|
||||
# The artifact stage depends on the check stage, so no tarball comes out of a build whose codecs fail.
|
||||
ARG BASE=public.ecr.aws/lambda/nodejs:22@sha256:d138eff445f6ea04c51091d8e6cbb064d5663e850cddfa62396061586f98b126
|
||||
|
||||
FROM ${BASE} AS build
|
||||
ENV SOURCE_DATE_EPOCH=1722643200
|
||||
RUN rpm -qa --qf '%{NAME}\n' | sort >/build-base-rpms.txt && \
|
||||
microdnf install -y --nodocs --setopt=install_weak_deps=0 --releasever=2023.12.20260914 \
|
||||
gcc gcc-c++ make cmake meson ninja-build autoconf automake libtool pkgconf-pkg-config diffutils \
|
||||
git xz bzip2 tar gzip which patchelf gnutls-devel zlib-devel $([ "$(uname -m)" = x86_64 ] && echo nasm) && \
|
||||
microdnf clean all && mkdir /build && mv /build-base-rpms.txt /build/base-rpms.txt
|
||||
COPY sources.txt build-ffmpeg.sh /build/
|
||||
RUN bash /build/build-ffmpeg.sh
|
||||
|
||||
FROM ${BASE} AS certs
|
||||
# certtool, not openssl: the base's openssl-libs cannot run the repo's openssl CLI.
|
||||
RUN microdnf install -y --nodocs --releasever=2023.12.20260914 gnutls-utils && mkdir /certs && cd /certs && \
|
||||
printf 'cn = check-ca\nca\ncert_signing_key\nexpiration_days = 2\n' >ca.tmpl && \
|
||||
printf 'cn = localhost\ndns_name = localhost\ntls_www_server\nexpiration_days = 2\n' >server.tmpl && \
|
||||
certtool -p --outfile ca.key && certtool -s --load-privkey ca.key --template ca.tmpl --outfile ca.crt && \
|
||||
certtool -p --outfile server.key && \
|
||||
certtool -c --load-privkey server.key --load-ca-certificate ca.crt --load-ca-privkey ca.key \
|
||||
--template server.tmpl --outfile server.crt
|
||||
|
||||
FROM ${BASE} AS check
|
||||
COPY --from=build /opt/hf-media /opt/hf-media
|
||||
COPY --from=certs /certs /certs
|
||||
COPY check-ffmpeg.sh /check-ffmpeg.sh
|
||||
RUN cp /certs/ca.crt /etc/pki/ca-trust/source/anchors/ && update-ca-trust && bash /check-ffmpeg.sh >/check.txt
|
||||
|
||||
FROM scratch AS artifact
|
||||
COPY --from=build /out/ /
|
||||
COPY --from=check /check.txt /hf-media-check.txt
|
||||
Executable
+92
@@ -0,0 +1,92 @@
|
||||
#!/bin/bash
|
||||
# Builds ffmpeg + ffprobe into /opt/hf-media and packs /out/hf-media-linux-<arch>.tar.gz.
|
||||
# Codec libraries link statically; glibc and the TLS stack stay dynamic (a static glibc build crashed on Lambda).
|
||||
set -euo pipefail
|
||||
|
||||
PREFIX=/opt/hf-media
|
||||
DEPS=/build/deps
|
||||
SRC=/build/src
|
||||
JOBS=$(nproc)
|
||||
export PKG_CONFIG_PATH="$DEPS/lib/pkgconfig:$DEPS/lib64/pkgconfig"
|
||||
export CFLAGS="-O2 -fPIC" CXXFLAGS="-O2 -fPIC"
|
||||
mkdir -p "$SRC" "$DEPS" "$PREFIX/share/licenses" /out
|
||||
|
||||
fetch() {
|
||||
local name=$1 sum url
|
||||
read -r sum url < <(awk -v n="$name" '$1 == n { print $2, $3 }' /build/sources.txt)
|
||||
curl -fsSL --retry 3 -o "$SRC/$name" "$url"
|
||||
echo "$sum $SRC/$name" | sha256sum -c --quiet
|
||||
tar -xf "$SRC/$name" -C "$SRC"
|
||||
}
|
||||
|
||||
license() { mkdir -p "$PREFIX/share/licenses/$1" && cp "${@:2}" "$PREFIX/share/licenses/$1/"; }
|
||||
|
||||
read -r x264_commit x264_url < <(awk '$1 == "x264" { print $2, $3 }' /build/sources.txt)
|
||||
git clone -q "$x264_url" "$SRC/x264"
|
||||
git -C "$SRC/x264" checkout -q "$x264_commit"
|
||||
[ "$(git -C "$SRC/x264" rev-parse HEAD)" = "$x264_commit" ]
|
||||
(cd "$SRC/x264" && ./configure --prefix="$DEPS" --enable-static --enable-pic --disable-cli &&
|
||||
make -j"$JOBS" && make install)
|
||||
license x264 "$SRC/x264/COPYING"
|
||||
|
||||
fetch x265_4.1.tar.gz
|
||||
# 8-bit x265 with the 10-bit library linked in (x265's multilib layout): HDR renders encode yuv420p10le.
|
||||
# SVE2 kernels need -march=armv9-a, which the base's gcc 11 does not know (x86 ignores the flag).
|
||||
x265=(-S "$SRC/x265_4.1/source" -DCMAKE_BUILD_TYPE=Release -DENABLE_SHARED=OFF -DENABLE_CLI=OFF -DENABLE_SVE2=OFF)
|
||||
cmake "${x265[@]}" -B "$SRC/x265-10" -DHIGH_BIT_DEPTH=ON -DEXPORT_C_API=OFF
|
||||
cmake --build "$SRC/x265-10" -j"$JOBS"
|
||||
cmake "${x265[@]}" -B "$SRC/x265-8" -DCMAKE_INSTALL_PREFIX="$DEPS" -DLINKED_10BIT=ON \
|
||||
-DEXTRA_LIB="$SRC/x265-10/libx265.a"
|
||||
cmake --build "$SRC/x265-8" -j"$JOBS"
|
||||
cmake --install "$SRC/x265-8"
|
||||
printf 'CREATE %s\nADDLIB %s\nADDLIB %s\nSAVE\nEND\n' "$DEPS/lib/libx265.a" \
|
||||
"$SRC/x265-8/libx265.a" "$SRC/x265-10/libx265.a" | ar -M
|
||||
license x265 "$SRC/x265_4.1/COPYING"
|
||||
|
||||
fetch libvpx-1.15.0.tar.gz
|
||||
(cd "$SRC/libvpx-1.15.0" && ./configure --prefix="$DEPS" --enable-static --disable-shared --enable-pic \
|
||||
--enable-vp9-highbitdepth --disable-examples --disable-tools --disable-docs --disable-unit-tests &&
|
||||
make -j"$JOBS" && make install)
|
||||
license libvpx "$SRC/libvpx-1.15.0/LICENSE" "$SRC/libvpx-1.15.0/PATENTS"
|
||||
|
||||
fetch opus-1.5.2.tar.gz
|
||||
(cd "$SRC/opus-1.5.2" && ./configure --prefix="$DEPS" --enable-static --disable-shared --with-pic \
|
||||
--disable-doc --disable-extra-programs && make -j"$JOBS" && make install)
|
||||
license opus "$SRC/opus-1.5.2/COPYING"
|
||||
|
||||
fetch zimg-3.0.5.tar.gz
|
||||
(cd "$SRC/zimg-release-3.0.5" && ./autogen.sh && ./configure --prefix="$DEPS" --enable-static --disable-shared \
|
||||
--with-pic && make -j"$JOBS" && make install)
|
||||
license zimg "$SRC/zimg-release-3.0.5/COPYING"
|
||||
|
||||
fetch dav1d-1.5.1.tar.xz
|
||||
meson setup "$SRC/dav1d-build" "$SRC/dav1d-1.5.1" --prefix="$DEPS" --libdir=lib --buildtype=release \
|
||||
--default-library=static -Denable_tools=false -Denable_tests=false
|
||||
ninja -C "$SRC/dav1d-build" install
|
||||
license dav1d "$SRC/dav1d-1.5.1/COPYING"
|
||||
|
||||
fetch ffmpeg-7.0.2.tar.xz
|
||||
# No --static pkg-config: gnutls.pc then names a libgmp.a from its own RPM build tree. The deps above
|
||||
# exist only as .a, so they link statically anyway; --extra-libs carries their private libraries.
|
||||
(cd "$SRC/ffmpeg-7.0.2" && ./configure --prefix="$PREFIX" \
|
||||
--extra-cflags="-I$DEPS/include" --extra-ldflags="-L$DEPS/lib" --extra-libs="-lstdc++ -lpthread -lm -ldl" \
|
||||
--enable-gpl --enable-version3 --disable-debug --disable-doc --disable-ffplay --disable-autodetect \
|
||||
--enable-gnutls --enable-zlib --enable-libx264 --enable-libx265 --enable-libvpx --enable-libopus \
|
||||
--enable-libzimg --enable-libdav1d && make -j"$JOBS" && make install)
|
||||
license ffmpeg "$SRC/ffmpeg-7.0.2/COPYING.GPLv3" "$SRC/ffmpeg-7.0.2/LICENSE.md"
|
||||
rm -rf "$PREFIX/include" "$PREFIX/lib" "$PREFIX/share/ffmpeg"
|
||||
strip "$PREFIX/bin/ffmpeg" "$PREFIX/bin/ffprobe"
|
||||
|
||||
# Bundle every shared library the bare base image lacks (gnutls, nettle); the rpath finds them.
|
||||
mkdir -p "$PREFIX/lib"
|
||||
ldd "$PREFIX/bin/ffmpeg" "$PREFIX/bin/ffprobe" | awk '$2 == "=>" { print $3 }' | sort -u | while read -r lib; do
|
||||
if ! grep -qx "$(rpm -qf --qf '%{NAME}' "$lib")" /build/base-rpms.txt; then
|
||||
cp -L "$lib" "$PREFIX/lib/" && patchelf --set-rpath '$ORIGIN' "$PREFIX/lib/$(basename "$lib")"
|
||||
echo "$(rpm -qf "$lib") $(basename "$lib")" >>"$PREFIX/share/licenses/bundled-libraries.txt"
|
||||
fi
|
||||
done
|
||||
patchelf --set-rpath '$ORIGIN/../lib' "$PREFIX/bin/ffmpeg" "$PREFIX/bin/ffprobe"
|
||||
cp /build/sources.txt "$PREFIX/share/licenses/sources.txt"
|
||||
|
||||
tar --sort=name --mtime="@$SOURCE_DATE_EPOCH" --owner=0 --group=0 --numeric-owner -C /opt -cf - hf-media |
|
||||
gzip -n -9 >"/out/hf-media-linux-$(uname -m).tar.gz"
|
||||
Executable
+61
@@ -0,0 +1,61 @@
|
||||
#!/bin/bash
|
||||
# Runs on the bare Lambda base with only /opt/hf-media added: every codec the engine calls must work.
|
||||
# Prints a deterministic summary to stdout; the artifact stage keeps it.
|
||||
set -euo pipefail
|
||||
ff() { /opt/hf-media/bin/ffmpeg -hide_banner -loglevel error -nostdin "$@"; }
|
||||
probe() { local input=$1; shift; /opt/hf-media/bin/ffprobe -hide_banner -loglevel error "$@" -- "$input"; }
|
||||
listed() { # listed <ffmpeg list flag> <awk field> <name...>
|
||||
local names
|
||||
names=$(/opt/hf-media/bin/ffmpeg -hide_banner "$1" | awk -v f="$2" '{ print $f }')
|
||||
for name in "${@:3}"; do grep -qx -- "$name" <<<"$names" || { echo "missing $1 $name" >&2; exit 1; }; done
|
||||
echo "${1#-}: ${*:3}"
|
||||
}
|
||||
frames() { probe "$1" -count_frames -select_streams v:0 -show_entries stream=nb_read_frames -of csv=p=0 "${@:2}"; }
|
||||
|
||||
cd "$(mktemp -d)"
|
||||
listed -encoders 2 libx264 libx265 libvpx-vp9 libopus prores_ks aac png mjpeg
|
||||
listed -decoders 2 libvpx-vp9 libvpx libdav1d h264 hevc prores aac opus png mjpeg
|
||||
listed -filters 2 zscale tonemap setparams ebur128 alphamerge overlay
|
||||
listed -protocols 1 https tls file
|
||||
|
||||
src=(-f lavfi -i "testsrc2=s=320x240:r=25:d=1")
|
||||
ff "${src[@]}" -c:v libx264 -pix_fmt yuv420p -bf 0 h264.mp4
|
||||
ff "${src[@]}" -c:v libx265 -pix_fmt yuv420p -tag:v hvc1 -x265-params log-level=error h265.mp4
|
||||
ff "${src[@]}" -c:v prores_ks -profile:v 4444 -pix_fmt yuva444p10le prores.mov
|
||||
ff -f lavfi -i "sine=d=1" -c:a libopus -b:a 128k opus.webm
|
||||
ff -f lavfi -i "sine=d=1" -c:a aac -b:a 192k aac.m4a
|
||||
ff "${src[@]}" -frames:v 1 frame.png
|
||||
for f in h264.mp4 h265.mp4 prores.mov; do [ "$(frames "$f")" = 25 ] || { echo "decode $f" >&2; exit 1; }; done
|
||||
for f in opus.webm aac.m4a; do ff -i "$f" -f null -; done
|
||||
[ "$(probe frame.png -show_entries stream=width -of csv=p=0)" = 320 ]
|
||||
echo "encode+decode: h264 h265 prores opus aac png"
|
||||
|
||||
# VP9 with alpha: left half transparent, decoded through libvpx-vp9 as the engine does.
|
||||
ff "${src[@]}" -vf "format=yuva420p,geq=lum='lum(X,Y)':cb='cb(X,Y)':cr='cr(X,Y)':a='if(lt(X,W/2),0,255)'" \
|
||||
-c:v libvpx-vp9 -pix_fmt yuva420p -auto-alt-ref 0 -b:v 0 -crf 30 alpha.webm
|
||||
ff -c:v libvpx-vp9 -i alpha.webm -frames:v 1 -vf alphaextract,crop=40:40:20:100 -f rawvideo -pix_fmt gray left.raw
|
||||
ff -c:v libvpx-vp9 -i alpha.webm -frames:v 1 -vf alphaextract,crop=40:40:260:100 -f rawvideo -pix_fmt gray right.raw
|
||||
node -e '
|
||||
const fs = require("fs"), mean = (f) => fs.readFileSync(f).reduce((a, b) => a + b, 0) / 1600;
|
||||
if (!(mean("left.raw") < 16 && mean("right.raw") > 240)) throw new Error("vp9 alpha lost");'
|
||||
echo "vp9 alpha: encode+decode"
|
||||
|
||||
# The engine's HDR to SDR chain (core/src/hdrToneMap.ts) on tagged 10-bit HEVC.
|
||||
ff "${src[@]}" -c:v libx265 -pix_fmt yuv420p10le -color_primaries bt2020 -color_trc smpte2084 \
|
||||
-colorspace bt2020nc -x265-params log-level=error hdr.mp4
|
||||
[ "$(probe hdr.mp4 -select_streams v:0 -show_entries stream=pix_fmt -of csv=p=0)" = yuv420p10le ]
|
||||
ff -i hdr.mp4 -vf "zscale=t=linear:npl=100,tonemap=hable:desat=0,zscale=p=bt709:t=bt709:m=bt709:r=tv" \
|
||||
-f null -
|
||||
echo "hdr: 10-bit hevc encode, zscale tonemap"
|
||||
|
||||
# TLS against the system trust store: a trusted host passes, a wrong host name is refused.
|
||||
exec 4< <(node -e '
|
||||
const fs = require("fs");
|
||||
require("https").createServer({ key: fs.readFileSync("/certs/server.key"), cert: fs.readFileSync("/certs/server.crt") },
|
||||
(_, res) => res.end(fs.readFileSync("h264.mp4"))).listen(8443, () => console.log("ready"));')
|
||||
server=$!
|
||||
trap 'kill $server' EXIT
|
||||
read -r -t 30 -u 4 ready && [ "$ready" = ready ]
|
||||
[ "$(frames https://localhost:8443/h264.mp4 -tls_verify 1)" = 25 ]
|
||||
if frames https://127.0.0.1:8443/h264.mp4 -tls_verify 1 2>/dev/null; then echo "tls accepted a wrong host" >&2; exit 1; fi
|
||||
echo "tls: verified, wrong host refused"
|
||||
@@ -0,0 +1,8 @@
|
||||
# name sha256 url (x264 has no stable tarballs: its line pins a git commit, r3191 as in the x86 ffmpeg-static)
|
||||
ffmpeg-7.0.2.tar.xz 8646515b638a3ad303e23af6a3587734447cb8fc0a0c064ecdb8e95c4fd8b389 https://ffmpeg.org/releases/ffmpeg-7.0.2.tar.xz
|
||||
x264 4613ac3c15fd75cebc4b9f65b7fb95e70a3acce1 https://code.videolan.org/videolan/x264.git
|
||||
x265_4.1.tar.gz a31699c6a89806b74b0151e5e6a7df65de4b49050482fe5ebf8a4379d7af8f29 https://bitbucket.org/multicoreware/x265_git/downloads/x265_4.1.tar.gz
|
||||
libvpx-1.15.0.tar.gz e935eded7d81631a538bfae703fd1e293aad1c7fd3407ba00440c95105d2011e https://github.com/webmproject/libvpx/archive/refs/tags/v1.15.0.tar.gz
|
||||
opus-1.5.2.tar.gz 65c1d2f78b9f2fb20082c38cbe47c951ad5839345876e46941612ee87f9a7ce1 https://downloads.xiph.org/releases/opus/opus-1.5.2.tar.gz
|
||||
zimg-3.0.5.tar.gz a9a0226bf85e0d83c41a8ebe4e3e690e1348682f6a2a7838f1b8cbff1b799bcf https://github.com/sekrit-twc/zimg/archive/refs/tags/release-3.0.5.tar.gz
|
||||
dav1d-1.5.1.tar.xz 401813f1f89fa8fd4295805aa5284d9aed9bc7fc1fdbe554af4292f64cbabe21 https://downloads.videolan.org/pub/videolan/dav1d/1.5.1/dav1d-1.5.1.tar.xz
|
||||
Reference in New Issue
Block a user