From 8cd9c517c79be5ada1878ba58bdffeac6cfe39eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81ngel?= Date: Tue, 27 Oct 2026 16:11:51 +0000 Subject: [PATCH] fix(aws-lambda): a network blip no longer stops the release ffmpeg build (#6569) --- packages/aws-lambda/image/Dockerfile | 8 ++-- packages/aws-lambda/image/build-ffmpeg.sh | 10 +---- packages/aws-lambda/image/fetch.sh | 19 +++++++++ packages/aws-lambda/image/fetch.test.ts | 52 +++++++++++++++++++++++ 4 files changed, 78 insertions(+), 11 deletions(-) create mode 100644 packages/aws-lambda/image/fetch.sh create mode 100644 packages/aws-lambda/image/fetch.test.ts diff --git a/packages/aws-lambda/image/Dockerfile b/packages/aws-lambda/image/Dockerfile index 36548885c8..007afd6ca0 100644 --- a/packages/aws-lambda/image/Dockerfile +++ b/packages/aws-lambda/image/Dockerfile @@ -5,8 +5,9 @@ ARG BASE=public.ecr.aws/lambda/nodejs:22@sha256:d138eff445f6ea04c51091d8e6cbb064 FROM ${BASE} AS build ENV SOURCE_DATE_EPOCH=1722643200 -RUN rpm -qa --qf '%{NAME}\n' | sort >/build-base-rpms.txt && \ - microdnf install -y --nodocs --setopt=install_weak_deps=0 --releasever=2023.12.20260914 \ +COPY fetch.sh /fetch.sh +RUN . /fetch.sh && rpm -qa --qf '%{NAME}\n' | sort >/build-base-rpms.txt && \ + retry microdnf install -y --nodocs --setopt=install_weak_deps=0 --releasever=2023.12.20260914 \ gcc gcc-c++ make cmake meson ninja-build autoconf automake libtool pkgconf-pkg-config diffutils \ git xz bzip2 tar gzip which patchelf gnutls-devel zlib-devel $([ "$(uname -m)" = x86_64 ] && echo nasm) && \ microdnf clean all && mkdir /build && mv /build-base-rpms.txt /build/base-rpms.txt @@ -14,8 +15,9 @@ COPY sources.txt build-ffmpeg.sh /build/ RUN bash /build/build-ffmpeg.sh FROM ${BASE} AS certs +COPY fetch.sh /fetch.sh # certtool, not openssl: the base's openssl-libs cannot run the repo's openssl CLI. -RUN microdnf install -y --nodocs --releasever=2023.12.20260914 gnutls-utils && mkdir /certs && cd /certs && \ +RUN . /fetch.sh && retry microdnf install -y --nodocs --releasever=2023.12.20260914 gnutls-utils && mkdir /certs && cd /certs && \ printf 'cn = check-ca\nca\ncert_signing_key\nexpiration_days = 2\n' >ca.tmpl && \ printf 'cn = localhost\ndns_name = localhost\ntls_www_server\nexpiration_days = 2\n' >server.tmpl && \ certtool -p --outfile ca.key && certtool -s --load-privkey ca.key --template ca.tmpl --outfile ca.crt && \ diff --git a/packages/aws-lambda/image/build-ffmpeg.sh b/packages/aws-lambda/image/build-ffmpeg.sh index 65492cfe22..fbbc222339 100755 --- a/packages/aws-lambda/image/build-ffmpeg.sh +++ b/packages/aws-lambda/image/build-ffmpeg.sh @@ -11,18 +11,12 @@ export PKG_CONFIG_PATH="$DEPS/lib/pkgconfig:$DEPS/lib64/pkgconfig" export CFLAGS="-O2 -fPIC" CXXFLAGS="-O2 -fPIC" mkdir -p "$SRC" "$DEPS" "$PREFIX/share/licenses" /out -fetch() { - local name=$1 sum url - read -r sum url < <(awk -v n="$name" '$1 == n { print $2, $3 }' /build/sources.txt) - curl -fsSL --retry 3 -o "$SRC/$name" "$url" - echo "$sum $SRC/$name" | sha256sum -c --quiet - tar -xf "$SRC/$name" -C "$SRC" -} +source /fetch.sh license() { mkdir -p "$PREFIX/share/licenses/$1" && cp "${@:2}" "$PREFIX/share/licenses/$1/"; } read -r x264_commit x264_url < <(awk '$1 == "x264" { print $2, $3 }' /build/sources.txt) -git clone -q "$x264_url" "$SRC/x264" +retry git clone -q "$x264_url" "$SRC/x264" git -C "$SRC/x264" checkout -q "$x264_commit" [ "$(git -C "$SRC/x264" rev-parse HEAD)" = "$x264_commit" ] (cd "$SRC/x264" && ./configure --prefix="$DEPS" --enable-static --enable-pic --disable-cli && diff --git a/packages/aws-lambda/image/fetch.sh b/packages/aws-lambda/image/fetch.sh new file mode 100644 index 0000000000..e09d84c290 --- /dev/null +++ b/packages/aws-lambda/image/fetch.sh @@ -0,0 +1,19 @@ +# Source downloads for the ffmpeg build. One place for the retry policy: curl alone does not retry a reset connection. +SOURCES=${SOURCES:-/build/sources.txt} + +retry() { + local n + for n in 1 2 3 4 5; do + "$@" && return 0 + sleep $((n * 3)) + done + return 1 +} + +fetch() { + local name=$1 sum url + read -r sum url <<<"$(awk -v n="$name" '$1 == n { print $2, $3 }' "$SOURCES")" + curl -fsSL --retry 5 --retry-all-errors --retry-delay 3 --connect-timeout 20 --max-time 600 -o "$SRC/$name" "$url" + echo "$sum $SRC/$name" | sha256sum -c --quiet + tar -xf "$SRC/$name" -C "$SRC" +} diff --git a/packages/aws-lambda/image/fetch.test.ts b/packages/aws-lambda/image/fetch.test.ts new file mode 100644 index 0000000000..5295b08fb7 --- /dev/null +++ b/packages/aws-lambda/image/fetch.test.ts @@ -0,0 +1,52 @@ +import { describe, expect, it } from "bun:test"; +import { createHash } from "node:crypto"; +import { mkdtempSync, readFileSync, writeFileSync } from "node:fs"; +import { createServer } from "node:net"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +const tarGz = (() => { + const dir = mkdtempSync(join(tmpdir(), "fetch-src-")); + writeFileSync(join(dir, "hello.txt"), "hi"); + const out = join(dir, "pkg.tar.gz"); + Bun.spawnSync(["tar", "-czf", out, "-C", dir, "hello.txt"]); + return readFileSync(out); +})(); + +async function fetchAfterResets(resets: number) { + let hits = 0; + const server = createServer((socket) => { + socket.once("data", () => { + if (++hits <= resets) return socket.resetAndDestroy(); + socket.write( + `HTTP/1.1 200 OK\r\nContent-Length: ${tarGz.length}\r\nConnection: close\r\n\r\n`, + ); + socket.end(tarGz); + }); + }).listen(0, "127.0.0.1"); + const { port } = server.address() as { port: number }; + const dir = mkdtempSync(join(tmpdir(), "fetch-run-")); + const sum = createHash("sha256").update(tarGz).digest("hex"); + writeFileSync( + join(dir, "sources.txt"), + `pkg.tar.gz ${sum} http://127.0.0.1:${port}/pkg.tar.gz\n`, + ); + const proc = Bun.spawn( + ["bash", "-c", `set -euo pipefail; source ${import.meta.dir}/fetch.sh; fetch pkg.tar.gz`], + { + env: { ...process.env, SRC: dir, SOURCES: join(dir, "sources.txt") }, + stdout: "pipe", + stderr: "pipe", + }, + ); + const code = await proc.exited; + server.close(); + return { code, hits, extracted: readFileSync(join(dir, "hello.txt"), "utf8") }; +} + +describe("ffmpeg image fetch", () => { + it("retries a reset connection and still checks the sha256", async () => { + const r = await fetchAfterResets(2); + expect(r).toMatchObject({ code: 0, hits: 3, extracted: "hi" }); + }, 30_000); +});