diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e691070f4a..d91a0eb127 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -264,6 +264,7 @@ jobs: - "scripts/package-subpaths.mjs" - "scripts/rewrite-relative-import-extensions.mjs" - "scripts/ci/base-file.sh" + - "scripts/ci/edit-accuracy-shard.sh" - "scripts/ci/install-workspace-dependencies.sh" - ".github/actions/prepare-ffmpeg-bin/**" - "package.json" @@ -1154,7 +1155,8 @@ jobs: needs: [changes, build] if: needs.changes.outputs.edit_accuracy == 'true' runs-on: ubuntu-latest - timeout-minutes: 45 + # Healthy shards take 6-9 min; 25 leaves a cancelled shard's fresh re-run inside the queue's 60 min check wait. + timeout-minutes: 25 strategy: fail-fast: false matrix: @@ -1182,15 +1184,14 @@ jobs: bash scripts/ci/base-file.sh packages/studio/tests/e2e/edit-accuracy/baseline.json /tmp/base-edit-accuracy.json [ -f /tmp/base-edit-accuracy.json ] || echo '{"cases":{}}' > /tmp/base-edit-accuracy.json - name: Run this shard, then re-run twice each case whose verdict differs from the base branch + run: bash scripts/ci/edit-accuracy-shard.sh "${{ matrix.shard }}/${{ strategy.job-total }}" + # A slow runner stops the bench (slow-runner.json) or runs into the timeout; either gets one fresh re-run. + - name: Mark a slow or cancelled shard for a re-run + if: always() run: | - set -euo pipefail - bench() { bun run --cwd packages/studio test:edit-accuracy -- --grid "$EDIT_ACCURACY_GRID" --jobs 2 "$@"; } - bench --shard "${{ matrix.shard }}/${{ strategy.job-total }}" --out /tmp/edit-accuracy/run1 - mapfile -t FLIPPED < <(node packages/studio/tests/e2e/edit-accuracy/ratchet.mjs flipped \ - /tmp/base-edit-accuracy.json /tmp/edit-accuracy/run1/results.json) - if (( ${#FLIPPED[@]} > 0 )); then - FILTER="^($(IFS='|'; echo "${FLIPPED[*]}"))$" - for run in 2 3; do bench --filter "$FILTER" --rerun --out "/tmp/edit-accuracy/run${run}"; done + mkdir -p /tmp/edit-accuracy + if [ "${{ job.status }}" = cancelled ] || [ -n "$(find /tmp/edit-accuracy -name slow-runner.json)" ]; then + touch /tmp/edit-accuracy/retry fi - name: Upload shard evidence if: always() @@ -1200,9 +1201,72 @@ jobs: path: /tmp/edit-accuracy if-no-files-found: error + studio-edit-accuracy-retry-plan: + name: "Studio: edit accuracy retry plan" + needs: [changes, studio-edit-accuracy] + if: ${{ !cancelled() && needs.changes.outputs.edit_accuracy == 'true' && needs.studio-edit-accuracy.result != 'success' }} + runs-on: ubuntu-latest + timeout-minutes: 5 + outputs: + shards: ${{ steps.plan.outputs.shards }} + steps: + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + pattern: edit-accuracy-shard-* + path: /tmp/edit-accuracy-shards + - name: List the shards marked for a re-run + id: plan + run: | + shards=$(find /tmp/edit-accuracy-shards -path '*/edit-accuracy-shard-*/retry' | + sed -E 's#.*/edit-accuracy-shard-([0-9]+)/retry$#\1#' | sort -n | paste -sd, -) + echo "shards=[${shards}]" >> "$GITHUB_OUTPUT" + + # Each marked shard runs once more on a fresh runner and replaces its evidence before the gate reads it. + studio-edit-accuracy-retry: + name: "Studio: edit accuracy retry (${{ matrix.shard }})" + needs: [changes, build, studio-edit-accuracy-retry-plan] + if: ${{ !cancelled() && needs.studio-edit-accuracy-retry-plan.outputs.shards != '[]' && needs.studio-edit-accuracy-retry-plan.outputs.shards != '' }} + runs-on: ubuntu-latest + timeout-minutes: 25 + strategy: + fail-fast: false + matrix: + shard: ${{ fromJSON(needs.studio-edit-accuracy-retry-plan.outputs.shards) }} + env: + EDIT_ACCURACY_GRID: ${{ github.event_name == 'merge_group' && 'full' || 'pr' }} + SHARDS: ${{ github.event_name == 'merge_group' && 20 || 5 }} + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + with: + node-version: 22 + - uses: ./.github/actions/prepare-ffmpeg-bin + - run: bash scripts/ci/install-workspace-dependencies.sh + - run: bun run --cwd packages/core build + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + name: edit-accuracy-cli + path: packages/cli/dist + - run: node packages/cli/dist/cli.js browser ensure + - name: Read the base branch's edit accuracy baseline + run: | + bash scripts/ci/base-file.sh packages/studio/tests/e2e/edit-accuracy/baseline.json /tmp/base-edit-accuracy.json + [ -f /tmp/base-edit-accuracy.json ] || echo '{"cases":{}}' > /tmp/base-edit-accuracy.json + - name: Run the shard again + run: bash scripts/ci/edit-accuracy-shard.sh "${{ matrix.shard }}/${SHARDS}" + - name: Replace the shard's evidence + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: edit-accuracy-shard-${{ matrix.shard }} + path: /tmp/edit-accuracy + if-no-files-found: error + overwrite: true + studio-edit-accuracy-gate: name: "Studio: edit accuracy gate" - needs: [changes, studio-edit-accuracy] + needs: [changes, studio-edit-accuracy, studio-edit-accuracy-retry] if: ${{ !cancelled() && needs.changes.outputs.edit_accuracy == 'true' }} runs-on: ubuntu-latest timeout-minutes: 5 diff --git a/package.json b/package.json index 3a533543f4..fbea196eb4 100644 --- a/package.json +++ b/package.json @@ -53,7 +53,7 @@ "player:perf": "bun run --filter @hyperframes/player perf", "format:check": "oxfmt --check .", "knip": "knip", - "test:scripts": "node --import tsx --test scripts/animejs-v4-guidance.test.mjs scripts/check-tracked-artifacts.test.mjs scripts/check-registry-set-delta.test.mjs scripts/check-no-main-deletions.test.mjs scripts/check-pr-captures.test.mjs scripts/check-comment-citations.test.mjs scripts/comments-workflow.test.mjs scripts/comment-ratchet.test.mjs scripts/check-docs-snippet-motion.test.mjs scripts/registry-target-paths.test.mjs scripts/check-workspace-contracts.test.mjs scripts/check-media-use-copy-parity.test.mjs scripts/check-svg-sanitize-parity.test.mjs scripts/check-media-use-svg-sanitize-generated.test.mjs scripts/check-package-cycles.test.mjs scripts/check-cli-process-ownership.test.mjs scripts/check-studio-copy-dashes.test.mjs scripts/check-large-files.test.mjs scripts/prune-fallow-audit-caches.test.mjs scripts/package-subpaths.test.mjs scripts/validate-release-channel.test.mjs scripts/clawhub-sync.test.mjs scripts/publish-workflow.test.mjs scripts/pr-edit-concurrency.test.mjs scripts/merge-queue-workflows.test.mjs scripts/install-workspace-dependencies.test.mjs scripts/check-edit-accuracy-filter.test.mjs scripts/draft-changelog.test.ts scripts/set-version.test.ts scripts/release-prepare.test.ts scripts/cli-options.test.ts scripts/changelog-weekly.test.ts scripts/claude-plugin-compression.test.ts scripts/catalog-payload-assets.test.ts scripts/host-registry-assets.test.ts scripts/catalog-preview-temp.test.ts scripts/backfill-block-previews.test.ts scripts/catalog-hosted-files.test.ts scripts/player-cdn-pin.test.ts scripts/studio-runtime-smoke.test.mjs scripts/verify-packed-manifests.test.mjs scripts/rewrite-relative-import-extensions.test.mjs scripts/lint-skills.test.mjs scripts/creator-editing-recipes.test.mjs packages/gcp-cloud-run/check-dockerfile-workspaces.test.mjs packages/core/scripts/writeGeneratedFile.test.ts scripts/catalog-publication.test.mjs scripts/ci/resolve-workflow-pr.test.mjs scripts/ci/base-file.test.mjs scripts/check-catalog-source-pr.test.mjs scripts/generate-registry-items.test.ts scripts/catalog-drift.test.ts scripts/catalog-fetch-mirror.test.ts scripts/catalog-script-inlining.test.ts scripts/catalog-detail.test.ts scripts/generate-catalog-pages.test.ts scripts/verify-catalog-payloads.test.ts scripts/registry-skill-files.test.ts scripts/creator-editing-capabilities.test.mjs scripts/generate-catalog-previews.test.ts scripts/registry-primitive-payloads.test.ts registry/components/pan-stations/pan-stations.test.mjs registry/components/filmed-screen/filmed-screen.test.mjs && vitest run scripts/catalog/ scripts/contrastRatchet.test.ts scripts/generate-catalog-payloads.test.ts", + "test:scripts": "node --import tsx --test scripts/animejs-v4-guidance.test.mjs scripts/check-tracked-artifacts.test.mjs scripts/check-registry-set-delta.test.mjs scripts/check-no-main-deletions.test.mjs scripts/check-pr-captures.test.mjs scripts/check-comment-citations.test.mjs scripts/comments-workflow.test.mjs scripts/comment-ratchet.test.mjs scripts/check-docs-snippet-motion.test.mjs scripts/registry-target-paths.test.mjs scripts/check-workspace-contracts.test.mjs scripts/check-media-use-copy-parity.test.mjs scripts/check-svg-sanitize-parity.test.mjs scripts/check-media-use-svg-sanitize-generated.test.mjs scripts/check-package-cycles.test.mjs scripts/check-cli-process-ownership.test.mjs scripts/check-studio-copy-dashes.test.mjs scripts/check-large-files.test.mjs scripts/prune-fallow-audit-caches.test.mjs scripts/package-subpaths.test.mjs scripts/validate-release-channel.test.mjs scripts/clawhub-sync.test.mjs scripts/publish-workflow.test.mjs scripts/pr-edit-concurrency.test.mjs scripts/merge-queue-workflows.test.mjs scripts/install-workspace-dependencies.test.mjs scripts/check-edit-accuracy-filter.test.mjs scripts/edit-accuracy-retry.test.mjs scripts/draft-changelog.test.ts scripts/set-version.test.ts scripts/release-prepare.test.ts scripts/cli-options.test.ts scripts/changelog-weekly.test.ts scripts/claude-plugin-compression.test.ts scripts/catalog-payload-assets.test.ts scripts/host-registry-assets.test.ts scripts/catalog-preview-temp.test.ts scripts/backfill-block-previews.test.ts scripts/catalog-hosted-files.test.ts scripts/player-cdn-pin.test.ts scripts/studio-runtime-smoke.test.mjs scripts/verify-packed-manifests.test.mjs scripts/rewrite-relative-import-extensions.test.mjs scripts/lint-skills.test.mjs scripts/creator-editing-recipes.test.mjs packages/gcp-cloud-run/check-dockerfile-workspaces.test.mjs packages/core/scripts/writeGeneratedFile.test.ts scripts/catalog-publication.test.mjs scripts/ci/resolve-workflow-pr.test.mjs scripts/ci/base-file.test.mjs scripts/check-catalog-source-pr.test.mjs scripts/generate-registry-items.test.ts scripts/catalog-drift.test.ts scripts/catalog-fetch-mirror.test.ts scripts/catalog-script-inlining.test.ts scripts/catalog-detail.test.ts scripts/generate-catalog-pages.test.ts scripts/verify-catalog-payloads.test.ts scripts/registry-skill-files.test.ts scripts/creator-editing-capabilities.test.mjs scripts/generate-catalog-previews.test.ts scripts/registry-primitive-payloads.test.ts registry/components/pan-stations/pan-stations.test.mjs registry/components/filmed-screen/filmed-screen.test.mjs && vitest run scripts/catalog/ scripts/contrastRatchet.test.ts scripts/generate-catalog-payloads.test.ts", "typecheck:scripts": "tsc --noEmit -p scripts/tsconfig.json", "test:skills": "HYPERFRAMES_MEDIA_HOME_REQUIRED=1 node --import ./packages/cli/scripts/test-home.mjs --test 'skills/**/*.test.mjs' 'packages/cli/src/media-use/**/*.test.mjs'", "generate:previews": "tsx scripts/generate-template-previews.ts", diff --git a/packages/studio/tests/e2e/edit-accuracy/report.mjs b/packages/studio/tests/e2e/edit-accuracy/report.mjs index ffb570756b..cfbdcf2f61 100644 --- a/packages/studio/tests/e2e/edit-accuracy/report.mjs +++ b/packages/studio/tests/e2e/edit-accuracy/report.mjs @@ -11,6 +11,14 @@ const lateUndo = (r) => Math.max(r.undo.ms ?? 0, r.undo.redoMs ?? 0) > UNDO_WRIT const DROPPED_FRAME_MS = 25; /** Smooth means no main-thread frame over this, and no more dropped frames than the blank-page control. */ const WORK_MS = 8; +/** A runner whose first cases take a median past this (3x a healthy runner's) is too slow to measure on. */ +export const SLOW_RUNNER = { cases: 4, medianSeconds: 60 }; +/** The first cases' median seconds when it marks a slow runner, else null; the blank-page control misses it. */ +export function slowRunner(seconds) { + if (seconds.length < SLOW_RUNNER.cases) return null; + const median = percentile(seconds.slice(0, SLOW_RUNNER.cases), 50); + return median > SLOW_RUNNER.medianSeconds ? median : null; +} export const METRICS = [ "tracking", "press", diff --git a/packages/studio/tests/e2e/edit-accuracy/report.test.mjs b/packages/studio/tests/e2e/edit-accuracy/report.test.mjs index a182b3518b..167e1bcd60 100644 --- a/packages/studio/tests/e2e/edit-accuracy/report.test.mjs +++ b/packages/studio/tests/e2e/edit-accuracy/report.test.mjs @@ -1,6 +1,6 @@ import { describe, expect, it } from "vitest"; import { accurate } from "./ratchet.mjs"; -import { entry, score, UNDO_WRITE_MAX_MS } from "./report.mjs"; +import { entry, score, slowRunner, UNDO_WRITE_MAX_MS } from "./report.mjs"; const frames = { intervals: [16, 17, 16], work: [2, 3, 2] }; const result = (undo) => ({ @@ -60,3 +60,20 @@ describe("a case's verdict", () => { expect(score({ id: "case" }, { error: "boom" }).pass).toBe(false); }); }); + +describe("the slow runner check", () => { + it("stops a shard whose first four cases ran about 6x slower than a healthy runner", () => { + // A merge queue shard that ran into its timeout: its first cases, in seconds. + expect(slowRunner([95.5, 133.7, 57.9, 107.5, 91.6])).toBe(95.5); + }); + + it("lets a healthy runner through, including one slow case among its first four", () => { + // The slowest first four of 20 healthy shards had a 22 s median; one healthy case took 92.7 s. + expect(slowRunner([22, 21.4, 19.8, 23.1])).toBeNull(); + expect(slowRunner([14, 92.7, 16.3, 15.2])).toBeNull(); + }); + + it("waits for four cases before judging", () => { + expect(slowRunner([95.5, 133.7, 107.5])).toBeNull(); + }); +}); diff --git a/packages/studio/tests/e2e/edit-accuracy/run.mjs b/packages/studio/tests/e2e/edit-accuracy/run.mjs index 5794cfe53f..1c5f37e531 100644 --- a/packages/studio/tests/e2e/edit-accuracy/run.mjs +++ b/packages/studio/tests/e2e/edit-accuracy/run.mjs @@ -14,7 +14,7 @@ import { resolveHeadlessShellPath } from "../../../../engine/src/index.ts"; import { buildGrid, writeFixture } from "./grid.mjs"; import { killServers, runCase, startServer, stopServer } from "./case.mjs"; import { runSequence } from "./sequences.mjs"; -import { METRICS, score, writeReport } from "./report.mjs"; +import { METRICS, SLOW_RUNNER, score, slowRunner, writeReport } from "./report.mjs"; import { holdProducerBrowser, renderBox } from "./render.mjs"; import { aabb, boxDistance } from "./geometry.mjs"; @@ -152,19 +152,28 @@ function acquireLock(path) { }); } +// Set once the first cases show a slow runner: no new case starts, and the run exits 75 (see the end). +let slow = null; + // The suite lock is taken per chunk, so a full grid never holds it for its whole run. const LOCK_CHUNK = 8; // fallow-ignore-next-line complexity async function runChunks(queue, browsers, decoders, results) { - for (let start = 0; start < queue.length; start += opt.lock ? LOCK_CHUNK : queue.length) { + for ( + let start = 0; + start < queue.length && slow === null; + start += opt.lock ? LOCK_CHUNK : queue.length + ) { const chunk = queue.slice(start, opt.lock ? start + LOCK_CHUNK : queue.length); const release = opt.lock ? await acquireLock(opt.lock) : () => undefined; try { await Promise.all( browsers.map(async (browser, i) => { - for (let spec = chunk.shift(); spec; spec = chunk.shift()) + for (let spec = chunk.shift(); spec && slow === null; spec = chunk.shift()) { results.push(await runOne(spec, browser, decoders[i], Number(opt.port) + i)); + slow = slowRunner(results.map((r) => r.seconds)); + } }), ); } finally { @@ -228,6 +237,18 @@ try { } finally { await Promise.all([releaseProducer(), ...browsers.map((b) => b.close())]); } +// CI reads slow-runner.json to re-run the shard once on a fresh runner. +if (slow !== null) { + const seconds = results.slice(0, SLOW_RUNNER.cases).map((r) => r.seconds); + writeFileSync( + join(out, "slow-runner.json"), + JSON.stringify({ seconds, median: slow, ...SLOW_RUNNER }), + ); + console.error( + `runner too slow: the first ${SLOW_RUNNER.cases} cases took a median ${slow.toFixed(1)}s, over ${SLOW_RUNNER.medianSeconds}s; stopping`, + ); + process.exit(75); +} results.sort((a, b) => a.id.localeCompare(b.id)); const meta = { studio, diff --git a/scripts/check-edit-accuracy-filter.test.mjs b/scripts/check-edit-accuracy-filter.test.mjs index 2c7dcc089b..5bf2124e24 100644 --- a/scripts/check-edit-accuracy-filter.test.mjs +++ b/scripts/check-edit-accuracy-filter.test.mjs @@ -180,11 +180,18 @@ class Closure { } } +// A shell script a step runs is read like a step: the scripts and commands it runs count too. +function seedRun(closure, path) { + const fresh = closure.isNew(path); + closure.add(path, true); + if (fresh && path.endsWith(".sh")) seed(closure, read(path)); +} + // Repo files and package scripts the jobs' steps name directly. function seed(closure, text) { for (const m of text.matchAll(/uses: \.\/(\S+)/g)) closure.add(`${m[1]}/action.yml`); for (const m of text.matchAll(/\b(?:bash|sh|node|bun) ((?:scripts|packages)\/\S+)/g)) - closure.add(m[1], true); + seedRun(closure, m[1]); for (const m of text.matchAll(/\bbun run --cwd (\S+) ([\w:-]+)/g)) closure.script(m[1], m[2]); } @@ -196,7 +203,12 @@ const cliCommands = (texts) => function benchInputs() { const closure = new Closure(); - const steps = ["build", "studio-edit-accuracy", "studio-edit-accuracy-gate"].map(job); + const steps = [ + "build", + "studio-edit-accuracy", + "studio-edit-accuracy-retry", + "studio-edit-accuracy-gate", + ].map(job); steps.forEach((text) => seed(closure, text)); for (const name of cliCommands([...steps, read(`${BENCH}/case.mjs`)])) closure.add(`packages/cli/src/commands/${name}.ts`); diff --git a/scripts/ci/edit-accuracy-shard.sh b/scripts/ci/edit-accuracy-shard.sh new file mode 100755 index 0000000000..a2c8fdb74a --- /dev/null +++ b/scripts/ci/edit-accuracy-shard.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +# edit-accuracy-shard.sh : runs one edit accuracy shard into /tmp/edit-accuracy, then re-runs twice each case +# whose verdict differs from the base branch (/tmp/base-edit-accuracy.json). A slow runner makes the bench exit 75. +set -euo pipefail +bench() { bun run --cwd packages/studio test:edit-accuracy -- --grid "$EDIT_ACCURACY_GRID" --jobs 2 "$@"; } +bench --shard "$1" --out /tmp/edit-accuracy/run1 +mapfile -t FLIPPED < <(node packages/studio/tests/e2e/edit-accuracy/ratchet.mjs flipped \ + /tmp/base-edit-accuracy.json /tmp/edit-accuracy/run1/results.json) +if (( ${#FLIPPED[@]} > 0 )); then + FILTER="^($(IFS='|'; echo "${FLIPPED[*]}"))$" + for run in 2 3; do bench --filter "$FILTER" --rerun --out "/tmp/edit-accuracy/run${run}"; done +fi diff --git a/scripts/edit-accuracy-retry.test.mjs b/scripts/edit-accuracy-retry.test.mjs new file mode 100644 index 0000000000..ed7f23b28a --- /dev/null +++ b/scripts/edit-accuracy-retry.test.mjs @@ -0,0 +1,103 @@ +import assert from "node:assert/strict"; +import { execFileSync } from "node:child_process"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; +import { parse } from "yaml"; + +// A slow runner must not drop a PR from the merge queue: its shard is marked, re-run once on a fresh runner, +// and the re-run's evidence is what the gate reads. +const ci = parse( + readFileSync(join(import.meta.dirname, "..", ".github", "workflows", "ci.yml"), "utf8"), +); +const jobs = ci.jobs; +const step = (job, name) => jobs[job].steps.find((s) => s.name === name); + +// Runs a step's shell with its fixed paths moved under root; returns what it wrote to GITHUB_OUTPUT. +function runStep(script, root, env = {}) { + const output = join(root, "github-output"); + writeFileSync(output, ""); + execFileSync("bash", ["-euo", "pipefail", "-c", script.replaceAll("/tmp/", `${root}/`)], { + env: { ...process.env, GITHUB_OUTPUT: output, ...env }, + }); + return readFileSync(output, "utf8"); +} + +function withTree(fn) { + const root = mkdtempSync(join(tmpdir(), "edit-accuracy-retry-")); + try { + return fn(root); + } finally { + rmSync(root, { recursive: true, force: true }); + } +} + +const markScript = (status) => + step("studio-edit-accuracy", "Mark a slow or cancelled shard for a re-run").run.replaceAll( + "${{ job.status }}", + status, + ); + +test("a shard is marked for a re-run when it timed out or its bench stopped on a slow runner", () => { + withTree((root) => { + runStep(markScript("cancelled"), root); + assert.ok(existsSync(join(root, "edit-accuracy/retry"))); + }); + withTree((root) => { + mkdirSync(join(root, "edit-accuracy/run1"), { recursive: true }); + writeFileSync(join(root, "edit-accuracy/run1/slow-runner.json"), "{}"); + runStep(markScript("failure"), root); + assert.ok(existsSync(join(root, "edit-accuracy/retry"))); + }); + withTree((root) => { + runStep(markScript("success"), root); + assert.ok(!existsSync(join(root, "edit-accuracy/retry"))); + }); +}); + +test("the retry plan lists exactly the marked shards", () => { + const plan = step("studio-edit-accuracy-retry-plan", "List the shards marked for a re-run").run; + withTree((root) => { + for (const shard of [7, 12, 3]) + mkdirSync(join(root, `edit-accuracy-shards/edit-accuracy-shard-${shard}`), { + recursive: true, + }); + writeFileSync(join(root, "edit-accuracy-shards/edit-accuracy-shard-7/retry"), ""); + writeFileSync(join(root, "edit-accuracy-shards/edit-accuracy-shard-12/retry"), ""); + assert.equal(runStep(plan, root), "shards=[7,12]\n"); + }); + withTree((root) => { + mkdirSync(join(root, "edit-accuracy-shards/edit-accuracy-shard-1"), { recursive: true }); + assert.equal(runStep(plan, root), "shards=[]\n"); + }); +}); + +test("each marked shard re-runs once and replaces its evidence before the gate reads it", () => { + const retry = jobs["studio-edit-accuracy-retry"]; + assert.match(retry.needs.join(" "), /studio-edit-accuracy-retry-plan/); + assert.match( + retry.strategy.matrix.shard, + /needs\.studio-edit-accuracy-retry-plan\.outputs\.shards/, + ); + const shardRun = retry.steps.find((s) => s.run?.includes("edit-accuracy-shard.sh")); + assert.ok(shardRun, "the retry runs the same shard script as the shard job"); + assert.ok( + jobs["studio-edit-accuracy"].steps.some((s) => + s.run?.includes("scripts/ci/edit-accuracy-shard.sh"), + ), + ); + const upload = retry.steps.find((s) => s.uses?.startsWith("actions/upload-artifact@")); + assert.equal(upload.with.name, "edit-accuracy-shard-${{ matrix.shard }}"); + assert.equal(upload.with.overwrite, true); + assert.ok(jobs["studio-edit-accuracy-gate"].needs.includes("studio-edit-accuracy-retry")); + assert.match(jobs["studio-edit-accuracy-gate"].if, /!cancelled\(\)/); +}); + +test("a cancelled shard and its re-run both fit in the merge queue's 60 minute check wait", () => { + const shard = jobs["studio-edit-accuracy"]["timeout-minutes"]; + const plan = jobs["studio-edit-accuracy-retry-plan"]["timeout-minutes"]; + const retry = jobs["studio-edit-accuracy-retry"]["timeout-minutes"]; + const gate = jobs["studio-edit-accuracy-gate"]["timeout-minutes"]; + assert.ok(shard + plan + retry + gate <= 60, `${shard}+${plan}+${retry}+${gate} min`); +});