test(studio): run the edit accuracy gate in CI across eight shards

Shares the built CLI from the Build job, re-runs twice each case that passes on the base branch and failed, and posts one sticky comment from the gate job.
This commit is contained in:
Miguel Ángel
2026-09-30 14:17:59 -07:00
parent d1daa952d0
commit 62167173cf
2 changed files with 134 additions and 0 deletions
+131
View File
@@ -74,6 +74,7 @@ jobs:
codex_plugin: ${{ steps.filter.outputs.codex_plugin }}
gcp_beginframe: ${{ steps.filter.outputs.gcp_beginframe }}
studio: ${{ steps.filter.outputs.studio }}
edit_accuracy: ${{ steps.filter.outputs.edit_accuracy }}
steps:
# Force git-based change detection instead of the pull_request REST API.
# The API path can fail the whole workflow on transient listFiles
@@ -158,6 +159,19 @@ jobs:
- "packages/studio-server/**"
- "bun.lock"
- ".github/workflows/ci.yml"
edit_accuracy:
- "packages/studio/src/components/editor/**"
- "packages/studio/src/components/nle/**"
- "packages/studio/src/hooks/**"
- "packages/studio/src/utils/**"
- "packages/studio/src/player/**"
- "packages/studio/src/webmcp/**"
- "packages/studio-server/src/**"
- "packages/core/src/**"
- "packages/cli/src/server/**"
- "packages/studio/tests/e2e/edit-accuracy/**"
- "bun.lock"
- ".github/workflows/ci.yml"
gcp_beginframe:
- "packages/gcp-cloud-run/Dockerfile"
- "packages/aws-lambda/scripts/probe-beginframe.ts"
@@ -216,6 +230,14 @@ jobs:
- run: bash scripts/ci/install-workspace-dependencies.sh
- run: bun run build
- run: bun run verify:packed-manifests
- name: Share the built CLI with the edit accuracy shards
if: github.event_name == 'pull_request' && needs.changes.outputs.edit_accuracy == 'true'
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
with:
name: edit-accuracy-cli
path: packages/cli/dist
if-no-files-found: error
retention-days: 1
lint:
name: Lint
@@ -924,6 +946,115 @@ jobs:
/tmp/studio-open-counts.json
if-no-files-found: error
# Manual editing accuracy in the built Studio, gated against the base branch's baseline.json.
studio-edit-accuracy:
name: "Studio: edit accuracy (${{ matrix.shard }}/8)"
needs: [changes, build]
if: github.event_name == 'pull_request' && needs.changes.outputs.edit_accuracy == 'true'
runs-on: ubuntu-latest
timeout-minutes: 45
strategy:
fail-fast: false
matrix:
shard: [1, 2, 3, 4, 5, 6, 7, 8]
steps:
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
with:
node-version: 22
- uses: ./.github/actions/prepare-ffmpeg-bin
- run: bash scripts/ci/install-workspace-dependencies.sh
- uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
with:
name: edit-accuracy-cli
path: packages/cli/dist
# Studio and the producer both run in the engine's pinned headless shell.
- run: node packages/cli/dist/cli.js browser ensure
- name: Read the base branch's edit accuracy baseline
env:
BASE_REF: ${{ github.base_ref }}
run: |
git fetch --no-tags --depth=1 origin "$BASE_REF"
if ! git show FETCH_HEAD:packages/studio/tests/e2e/edit-accuracy/baseline.json > /tmp/base-edit-accuracy.json; then
echo '{"cases":{}}' > /tmp/base-edit-accuracy.json
echo "The base branch has no edit accuracy baseline yet"
fi
- name: Run this shard, then re-run twice each case that passes on the base branch and failed
run: |
set -euo pipefail
bench() { bun run --cwd packages/studio test:edit-accuracy -- --grid full --jobs 2 "$@"; }
bench --shard "${{ matrix.shard }}/8" --out /tmp/edit-accuracy/run1
mapfile -t REGRESSED < <(node packages/studio/tests/e2e/edit-accuracy/ratchet.mjs regressions \
/tmp/base-edit-accuracy.json /tmp/edit-accuracy/run1/results.json)
if (( ${#REGRESSED[@]} > 0 )); then
FILTER="^($(IFS='|'; echo "${REGRESSED[*]}"))$"
for run in 2 3; do bench --filter "$FILTER" --rerun --out "/tmp/edit-accuracy/run${run}"; done
fi
- name: Upload shard evidence
if: always()
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
with:
name: edit-accuracy-shard-${{ matrix.shard }}
path: /tmp/edit-accuracy
if-no-files-found: error
studio-edit-accuracy-gate:
name: "Studio: edit accuracy gate"
needs: [changes, studio-edit-accuracy]
if: ${{ !cancelled() && github.event_name == 'pull_request' && needs.changes.outputs.edit_accuracy == 'true' }}
runs-on: ubuntu-latest
timeout-minutes: 5
permissions:
contents: read
pull-requests: write
steps:
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
with:
node-version: 22
- name: Read the base branch's edit accuracy baseline
env:
BASE_REF: ${{ github.base_ref }}
run: |
git fetch --no-tags --depth=1 origin "$BASE_REF"
if ! git show FETCH_HEAD:packages/studio/tests/e2e/edit-accuracy/baseline.json > /tmp/base-edit-accuracy.json; then
echo '{"cases":{}}' > /tmp/base-edit-accuracy.json
echo "The base branch has no edit accuracy baseline yet"
fi
- uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
with:
pattern: edit-accuracy-shard-*
path: /tmp/edit-accuracy-shards
- name: Compare every run with the base branch
id: gate
# Smoothness is reported, not gated; a case fails only if it fails 2 of its 3 runs.
run: |
set +e
mapfile -t RESULTS < <(find /tmp/edit-accuracy-shards -name results.json | sort)
node packages/studio/tests/e2e/edit-accuracy/ratchet.mjs gate /tmp/base-edit-accuracy.json \
packages/studio/tests/e2e/edit-accuracy/baseline.json /tmp/edit-accuracy-gate "${RESULTS[@]}"
echo "exit_code=$?" >> "$GITHUB_OUTPUT"
cat /tmp/edit-accuracy-gate/comment.md >> "$GITHUB_STEP_SUMMARY" || true
- name: Post sticky comment
# Fork PRs get a read-only token; the gate below still fires.
continue-on-error: true
uses: marocchino/sticky-pull-request-comment@773744901bac0e8cbb5a0dc842800d45e9b2b405 # v2.9.4
with:
header: edit-accuracy
path: /tmp/edit-accuracy-gate/comment.md
- name: Upload the gate report and a baseline.json to bank
if: always()
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
with:
name: edit-accuracy-gate
path: /tmp/edit-accuracy-gate
- name: Fail if the gate failed
if: steps.gate.outputs.exit_code != '0'
run: |
echo "::error::Edit accuracy fell against the base branch; see the PR comment."
exit 1
smoke-global-install:
name: "Smoke: global install"
needs: [changes, build]