ci(studio): re-run an edit accuracy shard on a fresh runner when its runner is too slow (#5659)

.github/workflows/ci.yml

@@ -264,6 +264,7 @@ jobs:
               - "scripts/package-subpaths.mjs"
               - "scripts/rewrite-relative-import-extensions.mjs"
               - "scripts/ci/base-file.sh"
+              - "scripts/ci/edit-accuracy-shard.sh"
               - "scripts/ci/install-workspace-dependencies.sh"
               - ".github/actions/prepare-ffmpeg-bin/**"
               - "package.json"
@@ -1154,7 +1155,8 @@ jobs:
     needs: [changes, build]
     if: needs.changes.outputs.edit_accuracy == 'true'
     runs-on: ubuntu-latest
-    timeout-minutes: 45
+    # Healthy shards take 6-9 min; 25 leaves a cancelled shard's fresh re-run inside the queue's 60 min check wait.
+    timeout-minutes: 25
     strategy:
       fail-fast: false
       matrix:
@@ -1182,15 +1184,14 @@ jobs:
           bash scripts/ci/base-file.sh packages/studio/tests/e2e/edit-accuracy/baseline.json /tmp/base-edit-accuracy.json
           [ -f /tmp/base-edit-accuracy.json ] || echo '{"cases":{}}' > /tmp/base-edit-accuracy.json
       - name: Run this shard, then re-run twice each case whose verdict differs from the base branch
+        run: bash scripts/ci/edit-accuracy-shard.sh "${{ matrix.shard }}/${{ strategy.job-total }}"
+      # A slow runner stops the bench (slow-runner.json) or runs into the timeout; either gets one fresh re-run.
+      - name: Mark a slow or cancelled shard for a re-run
+        if: always()
         run: |
-          set -euo pipefail
-          bench() { bun run --cwd packages/studio test:edit-accuracy -- --grid "$EDIT_ACCURACY_GRID" --jobs 2 "$@"; }
-          bench --shard "${{ matrix.shard }}/${{ strategy.job-total }}" --out /tmp/edit-accuracy/run1
-          mapfile -t FLIPPED < <(node packages/studio/tests/e2e/edit-accuracy/ratchet.mjs flipped \
-            /tmp/base-edit-accuracy.json /tmp/edit-accuracy/run1/results.json)
-          if (( ${#FLIPPED[@]} > 0 )); then
-            FILTER="^($(IFS='|'; echo "${FLIPPED[*]}"))$"
-            for run in 2 3; do bench --filter "$FILTER" --rerun --out "/tmp/edit-accuracy/run${run}"; done
+          mkdir -p /tmp/edit-accuracy
+          if [ "${{ job.status }}" = cancelled ] || [ -n "$(find /tmp/edit-accuracy -name slow-runner.json)" ]; then
+            touch /tmp/edit-accuracy/retry
           fi
       - name: Upload shard evidence
         if: always()
@@ -1200,9 +1201,72 @@ jobs:
           path: /tmp/edit-accuracy
           if-no-files-found: error
 
+  studio-edit-accuracy-retry-plan:
+    name: "Studio: edit accuracy retry plan"
+    needs: [changes, studio-edit-accuracy]
+    if: ${{ !cancelled() && needs.changes.outputs.edit_accuracy == 'true' && needs.studio-edit-accuracy.result != 'success' }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    outputs:
+      shards: ${{ steps.plan.outputs.shards }}
+    steps:
+      - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
+        with:
+          pattern: edit-accuracy-shard-*
+          path: /tmp/edit-accuracy-shards
+      - name: List the shards marked for a re-run
+        id: plan
+        run: |
+          shards=$(find /tmp/edit-accuracy-shards -path '*/edit-accuracy-shard-*/retry' |
+            sed -E 's#.*/edit-accuracy-shard-([0-9]+)/retry$#\1#' | sort -n | paste -sd, -)
+          echo "shards=[${shards}]" >> "$GITHUB_OUTPUT"
+
+  # Each marked shard runs once more on a fresh runner and replaces its evidence before the gate reads it.
+  studio-edit-accuracy-retry:
+    name: "Studio: edit accuracy retry (${{ matrix.shard }})"
+    needs: [changes, build, studio-edit-accuracy-retry-plan]
+    if: ${{ !cancelled() && needs.studio-edit-accuracy-retry-plan.outputs.shards != '[]' && needs.studio-edit-accuracy-retry-plan.outputs.shards != '' }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 25
+    strategy:
+      fail-fast: false
+      matrix:
+        shard: ${{ fromJSON(needs.studio-edit-accuracy-retry-plan.outputs.shards) }}
+    env:
+      EDIT_ACCURACY_GRID: ${{ github.event_name == 'merge_group' && 'full' || 'pr' }}
+      SHARDS: ${{ github.event_name == 'merge_group' && 20 || 5 }}
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+      - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
+      - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
+        with:
+          node-version: 22
+      - uses: ./.github/actions/prepare-ffmpeg-bin
+      - run: bash scripts/ci/install-workspace-dependencies.sh
+      - run: bun run --cwd packages/core build
+      - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
+        with:
+          name: edit-accuracy-cli
+          path: packages/cli/dist
+      - run: node packages/cli/dist/cli.js browser ensure
+      - name: Read the base branch's edit accuracy baseline
+        run: |
+          bash scripts/ci/base-file.sh packages/studio/tests/e2e/edit-accuracy/baseline.json /tmp/base-edit-accuracy.json
+          [ -f /tmp/base-edit-accuracy.json ] || echo '{"cases":{}}' > /tmp/base-edit-accuracy.json
+      - name: Run the shard again
+        run: bash scripts/ci/edit-accuracy-shard.sh "${{ matrix.shard }}/${SHARDS}"
+      - name: Replace the shard's evidence
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        with:
+          name: edit-accuracy-shard-${{ matrix.shard }}
+          path: /tmp/edit-accuracy
+          if-no-files-found: error
+          overwrite: true
+
   studio-edit-accuracy-gate:
     name: "Studio: edit accuracy gate"
-    needs: [changes, studio-edit-accuracy]
+    needs: [changes, studio-edit-accuracy, studio-edit-accuracy-retry]
     if: ${{ !cancelled() && needs.changes.outputs.edit_accuracy == 'true' }}
     runs-on: ubuntu-latest
     timeout-minutes: 5