ci(publish): the release keeps only Lambda assets npm pins or this run built (#7880)

The pack job no longer downloads Lambda assets already attached to the
release; it always pins the handler zip and video image this run built.

The publish job decides what the release keeps. Once
@hyperframes/aws-lambda is on npm for the version, the release must hold
exactly the assets that package pins. Before then, an asset this run did
not produce fails a first attempt, and a re-run of the same workflow run
replaces it. An asset identical to the bundle's is left alone, so a
re-run of only the publish job also works.
This commit is contained in:
Michael Wang
2026-11-16 03:45:25 +00:00
committed by GitHub
parent a0848ad353
commit 11a7d2749d
4 changed files with 236 additions and 93 deletions
+74 -55
View File
@@ -200,26 +200,19 @@ jobs:
name: hyperframes-lambda-handler-zip
path: dist/lambda-handler-build
# Before packing, so @hyperframes/aws-lambda pins this exact zip.
# Before packing, so @hyperframes/aws-lambda pins this exact zip. The pin
# is always this run's build; the publish job decides which asset the
# release keeps.
- name: Pin Lambda handler zip
id: lambda_zip
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
VERSION: ${{ steps.version.outputs.version }}
REPO: ${{ github.repository }}
EXPECTED_SHA256: ${{ needs.lambda-handler-zip.outputs.sha256 }}
run: |
# A retried run reuses the zip already on the release, so the pin still matches it.
if gh release download "v${VERSION}" --repo "$REPO" --pattern hyperframes-lambda-handler.zip --dir dist; then
echo "attached=true" >> "$GITHUB_OUTPUT"
else
ACTUAL_SHA256="$(sha256sum dist/lambda-handler-build/handler.zip | cut -d' ' -f1)"
if [ -z "$EXPECTED_SHA256" ] || [ "$ACTUAL_SHA256" != "$EXPECTED_SHA256" ]; then
echo "::error::Lambda handler zip sha256 $ACTUAL_SHA256 does not match the build job's $EXPECTED_SHA256"
exit 1
fi
mv dist/lambda-handler-build/handler.zip dist/hyperframes-lambda-handler.zip
ACTUAL_SHA256="$(sha256sum dist/lambda-handler-build/handler.zip | cut -d' ' -f1)"
if [ -z "$EXPECTED_SHA256" ] || [ "$ACTUAL_SHA256" != "$EXPECTED_SHA256" ]; then
echo "::error::Lambda handler zip sha256 $ACTUAL_SHA256 does not match the build job's $EXPECTED_SHA256"
exit 1
fi
mv dist/lambda-handler-build/handler.zip dist/hyperframes-lambda-handler.zip
rm -rf dist/lambda-handler-build
bun packages/aws-lambda/scripts/pin-handler-zip.ts
@@ -230,24 +223,14 @@ jobs:
# Built from the zip pinned above. The video stage only copies files, so no emulation is needed.
- name: Build Lambda video image
id: lambda_video_image
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
VERSION: ${{ steps.version.outputs.version }}
REPO: ${{ github.repository }}
run: |
# A retried run reuses the image already on the release, so the pin still matches it.
if gh release download "v${VERSION}" --repo "$REPO" --pattern hyperframes-lambda-video-arm64.oci.tar --dir dist; then
echo "attached=true" >> "$GITHUB_OUTPUT"
else
bun packages/aws-lambda/scripts/stage-arm64-image.ts --zip=dist/hyperframes-lambda-handler.zip \
--ffmpeg=dist/hf-media/hf-media-linux-aarch64.tar.gz --out=dist/arm64-image
# OCI export needs a docker-container builder; no attestations, so index.json lists one image.
docker buildx create --use
docker buildx build --progress=plain --platform linux/arm64 --target video \
--provenance=false --sbom=false --build-context task=dist/arm64-image/task \
--output type=oci,dest=dist/hyperframes-lambda-video-arm64.oci.tar packages/aws-lambda/image
fi
bun packages/aws-lambda/scripts/stage-arm64-image.ts --zip=dist/hyperframes-lambda-handler.zip \
--ffmpeg=dist/hf-media/hf-media-linux-aarch64.tar.gz --out=dist/arm64-image
# OCI export needs a docker-container builder; no attestations, so index.json lists one image.
docker buildx create --use
docker buildx build --progress=plain --platform linux/arm64 --target video \
--provenance=false --sbom=false --build-context task=dist/arm64-image/task \
--output type=oci,dest=dist/hyperframes-lambda-video-arm64.oci.tar packages/aws-lambda/image
bun packages/aws-lambda/scripts/pin-handler-zip.ts --video-image
- run: bun run verify:packed-manifests
@@ -275,21 +258,13 @@ jobs:
- name: Build portable agent plugin
run: node scripts/package-agent-plugin.mjs
# A Lambda asset the release already has stays out of the bundle; the
# publish job attaches only what the bundle holds.
- name: Collect release bundle
env:
VERSION: ${{ steps.version.outputs.version }}
ZIP_ATTACHED: ${{ steps.lambda_zip.outputs.attached }}
IMAGE_ATTACHED: ${{ steps.lambda_video_image.outputs.attached }}
run: |
mkdir -p release-bundle/assets release-bundle/plugin
if [ "$ZIP_ATTACHED" != "true" ]; then
mv dist/hyperframes-lambda-handler.zip dist/hyperframes-lambda-handler.sha256 release-bundle/assets/
fi
if [ "$IMAGE_ATTACHED" != "true" ]; then
mv dist/hyperframes-lambda-video-arm64.oci.tar dist/hyperframes-lambda-video-arm64.sha256 release-bundle/assets/
fi
mv dist/hyperframes-lambda-handler.zip dist/hyperframes-lambda-handler.sha256 release-bundle/assets/
mv dist/hyperframes-lambda-video-arm64.oci.tar dist/hyperframes-lambda-video-arm64.sha256 release-bundle/assets/
mv dist/hyperframes-agent-plugin.zip dist/hyperframes-agent-plugin.sha256 dist/hyperframes-agent-plugin.json release-bundle/plugin/
if [ -f "releases/v${VERSION}.md" ]; then
cp "releases/v${VERSION}.md" release-bundle/notes.md
@@ -516,27 +491,71 @@ jobs:
gh release create "v${VERSION}" "${FLAGS[@]}"
fi
- name: Attach Lambda handler zip to release
# Users get these assets only through the sha256 pins in
# @hyperframes/aws-lambda. Once that version is on npm, the release must
# keep exactly the pinned files. Before then, an asset this bundle did not
# produce fails a first attempt, because something other than this run
# attached it, and a re-run of this workflow run replaces it.
- name: Attach Lambda assets to release
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
VERSION: ${{ steps.version.outputs.version }}
REPO: ${{ github.repository }}
RUN_ATTEMPT: ${{ github.run_attempt }}
run: |
ASSETS="$RUNNER_TEMP/release-bundle/assets"
if [ -f "$ASSETS/hyperframes-lambda-handler.zip" ]; then
gh release upload "v${VERSION}" "$ASSETS/hyperframes-lambda-handler.zip" "$ASSETS/hyperframes-lambda-handler.sha256" --repo "$REPO"
META="$RUNNER_TEMP/aws-lambda-version.json"
PUBLISHED="$RUNNER_TEMP/aws-lambda-published.tgz"
rm -f "$PUBLISHED"
STATUS="$(curl -sS -o "$META" -w '%{http_code}' "https://registry.npmjs.org/@hyperframes%2faws-lambda/${VERSION}")"
if [ "$STATUS" = "200" ]; then
curl -fsSL -o "$PUBLISHED" "$(jq -r '.dist.tarball' "$META")"
INTEGRITY="sha512-$(openssl dgst -sha512 -binary "$PUBLISHED" | base64 -w0)"
if [ "$INTEGRITY" != "$(jq -r '.dist.integrity' "$META")" ]; then
echo "::error::The @hyperframes/aws-lambda@${VERSION} tarball does not match its npm integrity"
exit 1
fi
elif [ "$STATUS" != "404" ]; then
echo "::error::The npm registry answered $STATUS for @hyperframes/aws-lambda@${VERSION}"
exit 1
fi
ON_RELEASE="$RUNNER_TEMP/release-assets.tsv"
gh api "repos/$REPO/releases/tags/v${VERSION}" --jq '.assets[] | [.name, (.digest // "")] | @tsv' > "$ON_RELEASE"
- name: Attach Lambda video image to release
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
VERSION: ${{ steps.version.outputs.version }}
REPO: ${{ github.repository }}
run: |
ASSETS="$RUNNER_TEMP/release-bundle/assets"
if [ -f "$ASSETS/hyperframes-lambda-video-arm64.oci.tar" ]; then
gh release upload "v${VERSION}" "$ASSETS/hyperframes-lambda-video-arm64.oci.tar" "$ASSETS/hyperframes-lambda-video-arm64.sha256" --repo "$REPO"
fi
attach_asset() {
local asset="$1" sidecar="$2" manifest="$3"
local line attached pinned built
line="$(awk -F'\t' -v name="$asset" '$1 == name' "$ON_RELEASE")"
attached="${line#*$'\t'}"
if [ -n "$line" ] && [ -z "$attached" ]; then
echo "::error::Release v${VERSION} has $asset without a digest to check it by"
exit 1
fi
if [ -f "$PUBLISHED" ]; then
pinned="sha256:$(tar -xzOf "$PUBLISHED" "package/prebuilt/$manifest" | jq -r '.sha256')"
if [ "$attached" != "$pinned" ]; then
echo "::error::@hyperframes/aws-lambda@${VERSION} on npm pins $asset at $pinned, but release v${VERSION} has ${attached:-no such asset}"
exit 1
fi
echo "$asset on the release is the one npm pins — keeping it"
return 0
fi
built="sha256:$(sha256sum "$ASSETS/$asset" | cut -d' ' -f1)"
if [ -z "$line" ]; then
gh release upload "v${VERSION}" "$ASSETS/$asset" "$ASSETS/$sidecar" --repo "$REPO"
elif [ "$attached" = "$built" ]; then
echo "$asset is already attached with these contents — skipping"
elif [ "$RUN_ATTEMPT" -gt 1 ]; then
echo "::warning::Replacing $asset on release v${VERSION}; nothing on npm pins it yet."
gh release upload "v${VERSION}" "$ASSETS/$asset" "$ASSETS/$sidecar" --repo "$REPO" --clobber
else
echo "::error::Release v${VERSION} already has $asset, and this run did not attach it. Find out who did before releasing this version."
exit 1
fi
}
attach_asset hyperframes-lambda-handler.zip hyperframes-lambda-handler.sha256 handler-zip.json
attach_asset hyperframes-lambda-video-arm64.oci.tar hyperframes-lambda-video-arm64.sha256 video-image.json
# Runs outside the checkout so no project .npmrc applies, and with
# --ignore-scripts so no lifecycle script runs next to the npm token.
+1
View File
@@ -73,5 +73,6 @@ The publish workflow validates release channel boundaries before publishing:
- Builds, lifecycle scripts and `npm pack` run in a job with a read-only token and no secrets.
- The job that holds the npm token runs no code from the release commit. It checks that every packed tarball is a package this workflow publishes, at exactly the release version, then uploads it with lifecycle scripts off.
- A version already on npm is skipped only when its contents match the new tarball, or on a re-run of the same workflow run. Otherwise the publish fails, because something other than this run published that version.
- The Lambda handler zip and arm64 video image are always built by the run, and `@hyperframes/aws-lambda` pins those builds. Once that version is on npm, the release keeps exactly the assets it pins. Before then, if the release already holds a Lambda asset this run did not produce, a first attempt fails and a re-run of the same workflow run replaces the asset.
This prevents an alpha-only feature from being included in a stable hotfix by accident.
@@ -1,7 +1,7 @@
#!/usr/bin/env bun
// Release step: writes the handler ZIP's pin (asset URL + sha256) and the SAM template into the
// package, or with --video-image the arm64 video image's pin. A retried release run reuses the
// asset already on the release.
// package, or with --video-image the arm64 video image's pin. It always pins the file this run
// built.
import { createHash } from "node:crypto";
import { existsSync, mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";
import { dirname, join, resolve } from "node:path";
+159 -36
View File
@@ -186,21 +186,22 @@ test("the Lambda handler zip is pinned before packing and attached before npm pu
};
assert.ok(indexIn(pack, "Pin Lambda handler zip") < indexIn(pack, "Pack packages"));
assert.ok(
indexIn(publish, "Create GitHub Release") <
indexIn(publish, "Attach Lambda handler zip to release"),
indexIn(publish, "Create GitHub Release") < indexIn(publish, "Attach Lambda assets to release"),
);
assert.ok(
indexIn(publish, "Attach Lambda handler zip to release") < indexIn(publish, "Publish packages"),
indexIn(publish, "Attach Lambda assets to release") < indexIn(publish, "Publish packages"),
);
const attach = publish.steps[indexIn(publish, "Attach Lambda handler zip to release")];
assert.doesNotMatch(attach.run, /--clobber/);
});
test("the pin step reuses an attached zip and otherwise takes only the built zip's hash", () => {
test("the pack job pins only the Lambda assets this run built", () => {
for (const step of pack.steps) {
assert.doesNotMatch(step.run ?? "", /\bgh release\b/, step.name);
assert.ok(!JSON.stringify(step.env ?? {}).includes("secrets."), step.name);
}
const step = pack.steps.find((s) => s.name === "Pin Lambda handler zip");
const builtZip = "built handler zip";
const builtSha256 = createHash("sha256").update(builtZip).digest("hex");
const run = ({ ghExit, expectedSha256 }) => {
const run = (expectedSha256) => {
const root = mkdtempSync(join(tmpdir(), "hyperframes-lambda-zip-step-"));
try {
const bin = join(root, "bin");
@@ -209,24 +210,14 @@ test("the pin step reuses an attached zip and otherwise takes only the built zip
writeFileSync(join(root, "dist", "lambda-handler-build", "handler.zip"), builtZip);
const log = join(root, "calls.log");
writeFileSync(log, "");
for (const [name, exit] of [
["gh", ghExit],
["bun", 0],
]) {
writeFileSync(join(bin, name), `#!/bin/sh\necho "${name} $*" >> "${log}"\nexit ${exit}\n`);
chmodSync(join(bin, name), 0o755);
}
const output = join(root, "github-output");
writeFileSync(output, "");
writeFileSync(join(bin, "bun"), `#!/bin/sh\necho "bun $*" >> "${log}"\n`);
chmodSync(join(bin, "bun"), 0o755);
const result = spawnSync("bash", ["-euo", "pipefail", "-c", step.run], {
cwd: root,
encoding: "utf8",
env: {
...process.env,
PATH: `${bin}:${process.env.PATH}`,
GITHUB_OUTPUT: output,
VERSION: "9.8.7",
REPO: "o/r",
EXPECTED_SHA256: expectedSha256,
},
timeout: 5_000,
@@ -239,7 +230,6 @@ test("the pin step reuses an attached zip and otherwise takes only the built zip
status: result.status,
stdout: result.stdout,
calls: readFileSync(log, "utf8"),
output: readFileSync(output, "utf8"),
pinned,
};
} finally {
@@ -247,24 +237,13 @@ test("the pin step reuses an attached zip and otherwise takes only the built zip
}
};
const retried = run({ ghExit: 0, expectedSha256: builtSha256 });
assert.equal(retried.status, 0, retried.stdout);
assert.match(
retried.calls,
/^gh release download v9\.8\.7 --repo o\/r --pattern hyperframes-lambda-handler\.zip --dir dist$/m,
);
assert.match(retried.calls, /^bun packages\/aws-lambda\/scripts\/pin-handler-zip\.ts$/m);
assert.equal(retried.output, "attached=true\n");
const fresh = run({ ghExit: 1, expectedSha256: builtSha256 });
assert.equal(fresh.status, 0, fresh.stdout);
assert.doesNotMatch(fresh.calls, /build:zip/);
assert.match(fresh.calls, /^bun packages\/aws-lambda\/scripts\/pin-handler-zip\.ts$/m);
assert.equal(fresh.output, "");
assert.equal(fresh.pinned, builtZip);
const built = run(builtSha256);
assert.equal(built.status, 0, built.stdout);
assert.match(built.calls, /^bun packages\/aws-lambda\/scripts\/pin-handler-zip\.ts$/m);
assert.equal(built.pinned, builtZip);
for (const expectedSha256 of ["0".repeat(64), ""]) {
const rejected = run({ ghExit: 1, expectedSha256 });
const rejected = run(expectedSha256);
assert.equal(rejected.status, 1, rejected.stdout);
assert.match(rejected.stdout, /::error::Lambda handler zip sha256/);
assert.doesNotMatch(rejected.calls, /pin-handler-zip/);
@@ -272,6 +251,150 @@ test("the pin step reuses an attached zip and otherwise takes only the built zip
}
});
test("the release keeps only Lambda assets that npm pins or this run produced", () => {
const step = publish.steps.find((s) => s.name === "Attach Lambda assets to release");
assert.ok(step);
const sha = (text) => createHash("sha256").update(text).digest("hex");
const assets = {
"hyperframes-lambda-handler.zip": ["hyperframes-lambda-handler.sha256", "handler-zip.json"],
"hyperframes-lambda-video-arm64.oci.tar": [
"hyperframes-lambda-video-arm64.sha256",
"video-image.json",
],
};
const run = ({ npm = 404, pins, release = {}, attempt = 1, badIntegrity = false }) => {
const temp = mkdtempSync(join(tmpdir(), "hyperframes-attach-step-"));
try {
const bin = join(temp, "bin");
const bundle = join(temp, "release-bundle", "assets");
mkdirSync(bin);
mkdirSync(bundle, { recursive: true });
for (const [asset, [sidecar]] of Object.entries(assets)) {
writeFileSync(join(bundle, asset), `bundle ${asset}`);
writeFileSync(join(bundle, sidecar), `${sha(`bundle ${asset}`)} ${asset}\n`);
}
const fixture = join(temp, "fixture");
mkdirSync(join(fixture, "package", "prebuilt"), { recursive: true });
for (const [asset, [, manifest]] of Object.entries(assets)) {
const pinned = pins?.[asset] ?? `pinned ${asset}`;
writeFileSync(
join(fixture, "package", "prebuilt", manifest),
JSON.stringify({ sha256: sha(pinned) }),
);
}
const tarball = join(temp, "aws-lambda.tgz");
execFileSync("tar", ["-czf", tarball, "-C", fixture, "package"], { stdio: "pipe" });
const integrity = `sha512-${createHash("sha512").update(readFileSync(tarball)).digest("base64")}`;
writeFileSync(
join(temp, "meta.json"),
JSON.stringify({
dist: {
tarball: "https://registry.invalid/aws-lambda.tgz",
integrity: badIntegrity ? "sha512-other" : integrity,
},
}),
);
writeFileSync(
join(temp, "release.tsv"),
Object.entries(release)
.map(
([asset, contents]) =>
`${asset}\t${contents === null ? "" : `sha256:${sha(contents)}`}\n`,
)
.join(""),
);
const log = join(temp, "calls.log");
writeFileSync(log, "");
writeFileSync(
join(bin, "curl"),
[
"#!/bin/bash",
`echo "curl $*" >> "${log}"`,
'while [ $# -gt 0 ]; do case "$1" in -o) out="$2"; shift ;; -w) status=1 ;; esac; shift; done',
`if [ -n "$status" ]; then cp "${join(temp, "meta.json")}" "$out"; printf '%s' "${npm}"; else cp "${tarball}" "$out"; fi`,
].join("\n"),
);
writeFileSync(
join(bin, "gh"),
`#!/bin/sh\necho "gh $*" >> "${log}"\nif [ "$1" = api ]; then cat "${join(temp, "release.tsv")}"; fi\n`,
);
chmodSync(join(bin, "curl"), 0o755);
chmodSync(join(bin, "gh"), 0o755);
const result = runStep(step, {
cwd: temp,
path: `${bin}:${process.env.PATH}`,
env: {
RUNNER_TEMP: temp,
VERSION: "1.2.3",
REPO: "o/r",
GH_TOKEN: "test-token",
RUN_ATTEMPT: String(attempt),
},
});
const uploads = readFileSync(log, "utf8")
.split("\n")
.filter((line) => line.startsWith("gh release upload"));
return { ...result, uploads };
} finally {
rmSync(temp, { recursive: true, force: true });
}
};
const names = Object.keys(assets);
const bundleRelease = Object.fromEntries(names.map((name) => [name, `bundle ${name}`]));
const foreignRelease = Object.fromEntries(names.map((name) => [name, `other ${name}`]));
const pinnedRelease = Object.fromEntries(names.map((name) => [name, `pinned ${name}`]));
const fresh = run({});
assert.equal(fresh.status, 0, fresh.stdout + fresh.stderr);
assert.equal(fresh.uploads.length, 2);
for (const line of fresh.uploads) {
assert.match(line, /^gh release upload v1\.2\.3 \S+ \S+\.sha256 --repo o\/r$/);
}
const rerunPublish = run({ release: bundleRelease });
assert.equal(rerunPublish.status, 0, rerunPublish.stdout + rerunPublish.stderr);
assert.deepEqual(rerunPublish.uploads, []);
const foreign = run({ release: foreignRelease });
assert.equal(foreign.status, 1);
assert.match(
foreign.stdout,
/::error::Release v1\.2\.3 already has .* this run did not attach it/,
);
assert.deepEqual(foreign.uploads, []);
const rerunReplaces = run({ release: foreignRelease, attempt: 2 });
assert.equal(rerunReplaces.status, 0, rerunReplaces.stdout + rerunReplaces.stderr);
assert.equal(rerunReplaces.uploads.length, 2);
for (const line of rerunReplaces.uploads) assert.match(line, / --clobber$/);
assert.match(rerunReplaces.stdout, /::warning::Replacing/);
const keepsPinned = run({ npm: 200, release: pinnedRelease, attempt: 2 });
assert.equal(keepsPinned.status, 0, keepsPinned.stdout + keepsPinned.stderr);
assert.deepEqual(keepsPinned.uploads, []);
for (const release of [foreignRelease, bundleRelease, {}]) {
const unpinned = run({ npm: 200, release, attempt: 2 });
assert.equal(unpinned.status, 1, unpinned.stdout);
assert.match(unpinned.stdout, /::error::@hyperframes\/aws-lambda@1\.2\.3 on npm pins/);
assert.deepEqual(unpinned.uploads, []);
}
const tamperedNpm = run({ npm: 200, release: pinnedRelease, badIntegrity: true });
assert.equal(tamperedNpm.status, 1);
assert.match(tamperedNpm.stdout, /does not match its npm integrity/);
const registryDown = run({ npm: 503 });
assert.equal(registryDown.status, 1);
assert.match(registryDown.stdout, /npm registry answered 503/);
assert.deepEqual(registryDown.uploads, []);
const noDigest = run({ release: { [names[0]]: null } });
assert.equal(noDigest.status, 1);
assert.match(noDigest.stdout, /without a digest/);
assert.deepEqual(noDigest.uploads, []);
});
const stepOf = (job, name) => {
const step = job.steps.find((s) => s.name === name);
assert.ok(step, `missing step ${name}`);