From 62167173cf30720b3aaed36e6ba617f21678b2ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81ngel?= Date: Wed, 30 Sep 2026 05:45:12 -0400 Subject: [PATCH] test(studio): run the edit accuracy gate in CI across eight shards Shares the built CLI from the Build job, re-runs twice each case that passes on the base branch and failed, and posts one sticky comment from the gate job. --- .github/workflows/ci.yml | 131 ++++++++++++++++++ .../tests/e2e/edit-accuracy/ratchet.mjs | 3 + 2 files changed, 134 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 979886a970..35f14fb078 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -74,6 +74,7 @@ jobs: codex_plugin: ${{ steps.filter.outputs.codex_plugin }} gcp_beginframe: ${{ steps.filter.outputs.gcp_beginframe }} studio: ${{ steps.filter.outputs.studio }} + edit_accuracy: ${{ steps.filter.outputs.edit_accuracy }} steps: # Force git-based change detection instead of the pull_request REST API. # The API path can fail the whole workflow on transient listFiles @@ -158,6 +159,19 @@ jobs: - "packages/studio-server/**" - "bun.lock" - ".github/workflows/ci.yml" + edit_accuracy: + - "packages/studio/src/components/editor/**" + - "packages/studio/src/components/nle/**" + - "packages/studio/src/hooks/**" + - "packages/studio/src/utils/**" + - "packages/studio/src/player/**" + - "packages/studio/src/webmcp/**" + - "packages/studio-server/src/**" + - "packages/core/src/**" + - "packages/cli/src/server/**" + - "packages/studio/tests/e2e/edit-accuracy/**" + - "bun.lock" + - ".github/workflows/ci.yml" gcp_beginframe: - "packages/gcp-cloud-run/Dockerfile" - "packages/aws-lambda/scripts/probe-beginframe.ts" @@ -216,6 +230,14 @@ jobs: - run: bash scripts/ci/install-workspace-dependencies.sh - run: bun run build - run: bun run verify:packed-manifests + - name: Share the built CLI with the edit accuracy shards + if: github.event_name == 'pull_request' && needs.changes.outputs.edit_accuracy == 'true' + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: edit-accuracy-cli + path: packages/cli/dist + if-no-files-found: error + retention-days: 1 lint: name: Lint @@ -924,6 +946,115 @@ jobs: /tmp/studio-open-counts.json if-no-files-found: error + # Manual editing accuracy in the built Studio, gated against the base branch's baseline.json. + studio-edit-accuracy: + name: "Studio: edit accuracy (${{ matrix.shard }}/8)" + needs: [changes, build] + if: github.event_name == 'pull_request' && needs.changes.outputs.edit_accuracy == 'true' + runs-on: ubuntu-latest + timeout-minutes: 45 + strategy: + fail-fast: false + matrix: + shard: [1, 2, 3, 4, 5, 6, 7, 8] + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + with: + node-version: 22 + - uses: ./.github/actions/prepare-ffmpeg-bin + - run: bash scripts/ci/install-workspace-dependencies.sh + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + name: edit-accuracy-cli + path: packages/cli/dist + # Studio and the producer both run in the engine's pinned headless shell. + - run: node packages/cli/dist/cli.js browser ensure + - name: Read the base branch's edit accuracy baseline + env: + BASE_REF: ${{ github.base_ref }} + run: | + git fetch --no-tags --depth=1 origin "$BASE_REF" + if ! git show FETCH_HEAD:packages/studio/tests/e2e/edit-accuracy/baseline.json > /tmp/base-edit-accuracy.json; then + echo '{"cases":{}}' > /tmp/base-edit-accuracy.json + echo "The base branch has no edit accuracy baseline yet" + fi + - name: Run this shard, then re-run twice each case that passes on the base branch and failed + run: | + set -euo pipefail + bench() { bun run --cwd packages/studio test:edit-accuracy -- --grid full --jobs 2 "$@"; } + bench --shard "${{ matrix.shard }}/8" --out /tmp/edit-accuracy/run1 + mapfile -t REGRESSED < <(node packages/studio/tests/e2e/edit-accuracy/ratchet.mjs regressions \ + /tmp/base-edit-accuracy.json /tmp/edit-accuracy/run1/results.json) + if (( ${#REGRESSED[@]} > 0 )); then + FILTER="^($(IFS='|'; echo "${REGRESSED[*]}"))$" + for run in 2 3; do bench --filter "$FILTER" --rerun --out "/tmp/edit-accuracy/run${run}"; done + fi + - name: Upload shard evidence + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: edit-accuracy-shard-${{ matrix.shard }} + path: /tmp/edit-accuracy + if-no-files-found: error + + studio-edit-accuracy-gate: + name: "Studio: edit accuracy gate" + needs: [changes, studio-edit-accuracy] + if: ${{ !cancelled() && github.event_name == 'pull_request' && needs.changes.outputs.edit_accuracy == 'true' }} + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + pull-requests: write + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + with: + node-version: 22 + - name: Read the base branch's edit accuracy baseline + env: + BASE_REF: ${{ github.base_ref }} + run: | + git fetch --no-tags --depth=1 origin "$BASE_REF" + if ! git show FETCH_HEAD:packages/studio/tests/e2e/edit-accuracy/baseline.json > /tmp/base-edit-accuracy.json; then + echo '{"cases":{}}' > /tmp/base-edit-accuracy.json + echo "The base branch has no edit accuracy baseline yet" + fi + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + pattern: edit-accuracy-shard-* + path: /tmp/edit-accuracy-shards + - name: Compare every run with the base branch + id: gate + # Smoothness is reported, not gated; a case fails only if it fails 2 of its 3 runs. + run: | + set +e + mapfile -t RESULTS < <(find /tmp/edit-accuracy-shards -name results.json | sort) + node packages/studio/tests/e2e/edit-accuracy/ratchet.mjs gate /tmp/base-edit-accuracy.json \ + packages/studio/tests/e2e/edit-accuracy/baseline.json /tmp/edit-accuracy-gate "${RESULTS[@]}" + echo "exit_code=$?" >> "$GITHUB_OUTPUT" + cat /tmp/edit-accuracy-gate/comment.md >> "$GITHUB_STEP_SUMMARY" || true + - name: Post sticky comment + # Fork PRs get a read-only token; the gate below still fires. + continue-on-error: true + uses: marocchino/sticky-pull-request-comment@773744901bac0e8cbb5a0dc842800d45e9b2b405 # v2.9.4 + with: + header: edit-accuracy + path: /tmp/edit-accuracy-gate/comment.md + - name: Upload the gate report and a baseline.json to bank + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: edit-accuracy-gate + path: /tmp/edit-accuracy-gate + - name: Fail if the gate failed + if: steps.gate.outputs.exit_code != '0' + run: | + echo "::error::Edit accuracy fell against the base branch; see the PR comment." + exit 1 + smoke-global-install: name: "Smoke: global install" needs: [changes, build] diff --git a/packages/studio/tests/e2e/edit-accuracy/ratchet.mjs b/packages/studio/tests/e2e/edit-accuracy/ratchet.mjs index e5a53a59a1..705f1609d1 100644 --- a/packages/studio/tests/e2e/edit-accuracy/ratchet.mjs +++ b/packages/studio/tests/e2e/edit-accuracy/ratchet.mjs @@ -28,6 +28,7 @@ const summary = (e) => e.error ? "error" : `${GATED_PX.map((m) => `${m} ${e[m] ?? "-"}`).join(", ")}, undo ${e.undo}`; /** Every run of every case: each shard's run plus the re-runs of its regressions. */ +// fallow-ignore-next-line complexity export function gate(base, head, runs) { const seen = new Map(); for (const r of runs) seen.set(r.id, [...(seen.get(r.id) ?? []), entry(r)]); @@ -108,6 +109,7 @@ const readBaseline = (path) => { } }; +// fallow-ignore-next-line complexity function main([command, basePath, ...rest]) { const base = readBaseline(basePath); if (command === "regressions") { @@ -116,6 +118,7 @@ function main([command, basePath, ...rest]) { } const [headPath, out, ...resultPaths] = rest; const runs = resultPaths.map((p) => readJson(p)); + if (!runs.length) throw new Error("no results.json from any shard"); const g = gate( base, readBaseline(headPath),