diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 1c64ae1bcc..98a15d8bbe 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -31,8 +31,37 @@ concurrency: cancel-in-progress: false jobs: + release-source: + name: Check the release commit + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + if: >- + github.event_name == 'push' || + (github.event.pull_request.merged == true && + startsWith(github.event.pull_request.head.ref, 'release/v')) + steps: + # A prerelease tag can name any commit; every job after this builds what it names. Alphas are cut from main, + # next or alpha (release-channels.mdx), so the tagged commit must already be on one of them. + - name: Check the tagged commit is on main, next or alpha + if: github.event_name == 'push' + env: + GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} + TAGGED: ${{ github.sha }} + run: | + for branch in main next alpha; do + # A branch that does not exist answers 404; that branch holds nothing. + status="$(gh api "repos/$REPO/compare/$branch...$TAGGED" --jq .status 2>/dev/null)" || continue + case "$status" in identical | behind) exit 0 ;; esac + done + echo "::error::the tagged commit $TAGGED is not an ancestor of main, next or alpha, so it does not publish" + exit 1 + lambda-arm64-ffmpeg: name: Lambda arm64 ffmpeg + needs: [release-source] runs-on: ubuntu-24.04-arm # A cache miss compiles ffmpeg natively, 15-20 minutes. timeout-minutes: 60 @@ -74,6 +103,7 @@ jobs: lambda-handler-zip: name: Lambda handler zip + needs: [release-source] runs-on: ubuntu-latest timeout-minutes: 30 # Runs apart from the publish job: read-only token, no secrets, no stored git credentials. @@ -429,43 +459,6 @@ jobs: exit 1 fi - - name: Create release tag - if: github.event_name == 'pull_request' - env: - VERSION: ${{ steps.version.outputs.version }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - : "${GITHUB_TOKEN:?}" - BASIC_AUTH="$(printf 'x-access-token:%s' "$GITHUB_TOKEN" | base64 -w0)" - echo "::add-mask::$BASIC_AUTH" - export GIT_CONFIG_COUNT=1 - export GIT_CONFIG_KEY_0="http.https://github.com/.extraheader" - export GIT_CONFIG_VALUE_0="AUTHORIZATION: basic $BASIC_AUTH" - - TAG="v$VERSION" - EXPECTED_TAG_SHA="$(git rev-parse HEAD)" - - verify_remote_tag() { - git fetch --force --no-tags origin "+refs/tags/$TAG:refs/tags/$TAG" - ACTUAL_TAG_SHA="$(git rev-parse "refs/tags/$TAG^{commit}")" - if [ "$ACTUAL_TAG_SHA" != "$EXPECTED_TAG_SHA" ]; then - echo "::error::Release tag $TAG points to $ACTUAL_TAG_SHA, expected $EXPECTED_TAG_SHA" - exit 1 - fi - echo "Release tag $TAG already exists at the expected commit — skipping" - } - - if [ -n "$(git ls-remote --refs origin "refs/tags/$TAG")" ]; then - verify_remote_tag - else - git tag --no-sign "$TAG" "$EXPECTED_TAG_SHA" - if ! git push origin "refs/tags/$TAG"; then - # A concurrent retry may have created the tag after ls-remote. - git tag -d "$TAG" - verify_remote_tag - fi - fi - - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 with: node-version: 24 @@ -544,6 +537,76 @@ jobs: fi cat "$ORDER" + # npm publish reads each tarball with its own node-tar; a header GNU tar reads another way can give it + # a different package.json than the one checked above. This asks that npm, without uploading, what it publishes. + - name: Check what npm will publish + working-directory: ${{ runner.temp }} + env: + VERSION: ${{ steps.version.outputs.version }} + DIST_TAG: ${{ steps.version.outputs.dist_tag }} + run: | + while IFS=$'\t' read -r name tarball; do + if ! view="$(npm publish "$tarball" --dry-run --json --ignore-scripts --access public --tag "$DIST_TAG" \ + --registry https://registry.npmjs.org/ 2>/dev/null)"; then + echo "::error::npm cannot read $(basename "$tarball")" + exit 1 + fi + # npm 10 prints one object, npm 11 keys it by package name. + read_as="$(jq -r 'if has("name") then . else (to_entries | if length == 1 then .[0].value else error("more than one package") end) end + | "\(.name)@\(.version)\t\([.files[].path | select(. != "")] | sort | join("\n"))"' <<<"$view")" || { + echo "::error::npm reads $(basename "$tarball") as more than one package" + exit 1 + } + npm_id="${read_as%%$'\t'*}" + npm_files="${read_as#*$'\t'}" + tar_files="$(tar -tzf "$tarball" | grep -v '/$' | sed 's#^package/##' | LC_ALL=C sort)" + if [ "$npm_id" != "$name@$VERSION" ]; then + echo "::error::npm would publish $(basename "$tarball") as $npm_id, not $name@$VERSION" + exit 1 + fi + if [ "$npm_files" != "$(LC_ALL=C sort <<<"$npm_files")" ] || [ "$npm_files" != "$tar_files" ]; then + echo "::error::npm and tar read different files in $(basename "$tarball")" + exit 1 + fi + done < publish-order.tsv + + - name: Create release tag + if: github.event_name == 'pull_request' + env: + VERSION: ${{ steps.version.outputs.version }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + : "${GITHUB_TOKEN:?}" + BASIC_AUTH="$(printf 'x-access-token:%s' "$GITHUB_TOKEN" | base64 -w0)" + echo "::add-mask::$BASIC_AUTH" + export GIT_CONFIG_COUNT=1 + export GIT_CONFIG_KEY_0="http.https://github.com/.extraheader" + export GIT_CONFIG_VALUE_0="AUTHORIZATION: basic $BASIC_AUTH" + + TAG="v$VERSION" + EXPECTED_TAG_SHA="$(git rev-parse HEAD)" + + verify_remote_tag() { + git fetch --force --no-tags origin "+refs/tags/$TAG:refs/tags/$TAG" + ACTUAL_TAG_SHA="$(git rev-parse "refs/tags/$TAG^{commit}")" + if [ "$ACTUAL_TAG_SHA" != "$EXPECTED_TAG_SHA" ]; then + echo "::error::Release tag $TAG points to $ACTUAL_TAG_SHA, expected $EXPECTED_TAG_SHA" + exit 1 + fi + echo "Release tag $TAG already exists at the expected commit — skipping" + } + + if [ -n "$(git ls-remote --refs origin "refs/tags/$TAG")" ]; then + verify_remote_tag + else + git tag --no-sign "$TAG" "$EXPECTED_TAG_SHA" + if ! git push origin "refs/tags/$TAG"; then + # A concurrent retry may have created the tag after ls-remote. + git tag -d "$TAG" + verify_remote_tag + fi + fi + - name: Check published Lambda pins env: VERSION: ${{ steps.version.outputs.version }} diff --git a/docs/contributing/release-channels.mdx b/docs/contributing/release-channels.mdx index dd2cc3c061..fd2e5b6cd0 100644 --- a/docs/contributing/release-channels.mdx +++ b/docs/contributing/release-channels.mdx @@ -40,7 +40,7 @@ For hotfixes, branch from the last stable tag, cherry-pick only the fix, publish ## Alpha release -Alpha releases must be reachable from a prerelease branch such as `origin/next` or `origin/alpha`. +Alpha releases must be reachable from `origin/main`, `origin/next` or `origin/alpha`. The publish workflow refuses a tag on any other commit before it builds anything. Use the same changelog draft workflow when the prerelease contains changes that users should know about. ```bash @@ -66,7 +66,7 @@ The publish workflow validates release channel boundaries before publishing: - Stable versions must publish with `latest`. - Prerelease versions must publish with the prerelease dist-tag, such as `alpha`. - Stable releases must come from a merged, reviewed `release/vX.Y.Z` PR. -- Prerelease tags must be reachable from a prerelease branch. +- Prerelease tags must point at a commit already on `main`, `next` or `alpha`. - Every publish, stable or prerelease, waits for a maintainer other than the one who started it to approve the run in the `npm-publish` environment. - Merged `release/vX.Y.Z` PRs publish stable releases only. - Every publish job verifies that its checkout matches the immutable event SHA. diff --git a/scripts/publish-workflow.test.mjs b/scripts/publish-workflow.test.mjs index afefd4e40c..5473abce49 100644 --- a/scripts/publish-workflow.test.mjs +++ b/scripts/publish-workflow.test.mjs @@ -12,6 +12,7 @@ import { } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; +import { gzipSync } from "node:zlib"; import test from "node:test"; import { parse } from "yaml"; @@ -609,3 +610,142 @@ for (const [job, name, artifact] of [ assert.notEqual(runFindPrevious(step, [{ id: 1, name: artifact, expired: true }]).status, 0); }); } + +// A tarball whose second entry GNU tar reads as package/d/package/package.json and npm's node-tar reads as +// package/package.json: ustar magic, version bytes other than "00", a non-empty prefix. +function packSplitReading(dir, file, checked, published) { + const staging = mkdtempSync(join(tmpdir(), "hyperframes-split-fixture-")); + try { + mkdirSync(join(staging, "package")); + writeFileSync(join(staging, "package", "package.json"), JSON.stringify(checked)); + writeFileSync(join(staging, "package", "x.json"), JSON.stringify(published)); + const tar = execFileSync("tar", [ + "--format=ustar", + "-cf", + "-", + "-C", + staging, + "package/package.json", + "package/x.json", + ]); + const header = 1024; + tar.fill(0, header, header + 100); + tar.write("package/package.json", header, "latin1"); + tar.fill(0, header + 345, header + 500); + tar.write("package/d", header + 345, "latin1"); + tar.write("ustar\0", header + 257, "latin1"); + tar.fill(0, header + 263, header + 265); + tar.fill(0x20, header + 148, header + 156); + let sum = 0; + for (let i = header; i < header + 512; i += 1) sum += tar[i]; + tar.write(`${sum.toString(8).padStart(6, "0")}\0 `, header + 148, "latin1"); + writeFileSync(join(dir, file), gzipSync(tar)); + } finally { + rmSync(staging, { recursive: true, force: true }); + } +} + +test("npm publishes only what the check read: name, version and every file", () => { + const step = stepOf(publish, "Check what npm will publish"); + const name = "@hyperframes/core"; + const version = "1.2.3-alpha.1"; + const run = (make, expected = name) => { + const temp = mkdtempSync(join(tmpdir(), "hyperframes-npm-reads-")); + try { + make(temp, "core.tgz"); + writeFileSync(join(temp, "publish-order.tsv"), `${expected}\t${join(temp, "core.tgz")}\n`); + return runStep(step, { cwd: temp, env: { VERSION: version, DIST_TAG: "alpha" } }); + } finally { + rmSync(temp, { recursive: true, force: true }); + } + }; + const clean = run((dir, file) => packTarball(dir, file, { name, version })); + assert.equal(clean.status, 0, clean.stderr + clean.stdout); + // fails without the fix: the layout check passed this tarball and npm would publish swapped-package@9.9.9. + const split = run((dir, file) => + packSplitReading(dir, file, { name, version }, { name: "swapped-package", version: "9.9.9" }), + ); + assert.equal(split.status, 1); + assert.match( + split.stdout, + /npm (would publish core\.tgz as swapped-package@9\.9\.9|and tar read different files)/, + ); + // Same name and version, other contents: only the file lists differ. + const swapped = run((dir, file) => + packSplitReading( + dir, + file, + { name, version }, + { name, version, scripts: { postinstall: "x" } }, + ), + ); + assert.equal(swapped.status, 1); + assert.match(swapped.stdout, /npm and tar read different files in core\.tgz/); + const renamed = run((dir, file) => + packTarball(dir, file, { name: "@hyperframes/other", version }), + ); + assert.equal(renamed.status, 1); + assert.match( + renamed.stdout, + /would publish core\.tgz as @hyperframes\/other@1\.2\.3-alpha\.1, not @hyperframes\/core@1\.2\.3-alpha\.1/, + ); +}); + +const stepIndex = (job, name) => { + const index = job.steps.findIndex((step) => step.name === name); + assert.ok(index >= 0, `missing step ${name}`); + return index; +}; + +test("both tarball checks pass before the release tag or anything else is written", () => { + const tag = stepIndex(publish, "Create release tag"); + assert.ok(stepIndex(publish, "Check packed tarballs") < tag); + assert.ok(stepIndex(publish, "Check what npm will publish") < tag); + assert.ok( + stepIndex(publish, "Check packed tarballs") < stepIndex(publish, "Check what npm will publish"), + ); +}); + +test("a tag publishes only a commit already on main, next or alpha, before any job builds it", () => { + const source = config.jobs["release-source"]; + assert.ok(source, "missing the release-source job"); + for (const name of ["lambda-arm64-ffmpeg", "lambda-handler-zip"]) { + assert.deepEqual(config.jobs[name].needs, ["release-source"], name); + } + assert.deepEqual(pack.needs.sort(), ["lambda-arm64-ffmpeg", "lambda-handler-zip"]); + const step = stepOf(source, "Check the tagged commit is on main, next or alpha"); + assert.equal(step.if, "github.event_name == 'push'"); + const bin = mkdtempSync(join(tmpdir(), "hyperframes-fake-gh-")); + try { + // The fake answers compare/... from FAKE_; an unset branch answers like a 404. + writeFileSync( + join(bin, "gh"), + '#!/bin/sh\nb="${2#*compare/}"; b="${b%%...*}"\neval "v=\\${FAKE_$b-}"\n[ -n "$v" ] || exit 1\necho "$v"\n', + ); + chmodSync(join(bin, "gh"), 0o755); + const status = (branches) => + runStep(step, { + cwd: bin, + env: { + REPO: "heygen-com/hyperframes", + TAGGED: "abc", + ...Object.fromEntries(Object.entries(branches).map(([b, v]) => [`FAKE_${b}`, v])), + }, + path: `${bin}:${process.env.PATH}`, + }).status; + assert.equal(status({ main: "identical" }), 0); + assert.equal(status({ main: "behind" }), 0); + assert.equal( + status({ main: "diverged", next: "behind" }), + 0, + "a commit on next publishes an alpha", + ); + assert.equal(status({ main: "ahead", next: "diverged", alpha: "identical" }), 0); + // fails without the fix: a v*-* tag on any other commit went on to build and publish. + assert.equal(status({ main: "ahead", next: "diverged" }), 1, "a commit on no allowed branch"); + assert.equal(status({ main: "diverged", next: "ahead", alpha: "ahead" }), 1); + assert.equal(status({}), 1, "no branch answers"); + } finally { + rmSync(bin, { recursive: true, force: true }); + } +});