ci(studio): re-run an edit accuracy shard on a fresh runner when its runner is too slow (#5659)

scripts/check-edit-accuracy-filter.test.mjs

@@ -180,11 +180,18 @@ class Closure {
   }
 }
 
+// A shell script a step runs is read like a step: the scripts and commands it runs count too.
+function seedRun(closure, path) {
+  const fresh = closure.isNew(path);
+  closure.add(path, true);
+  if (fresh && path.endsWith(".sh")) seed(closure, read(path));
+}
+
 // Repo files and package scripts the jobs' steps name directly.
 function seed(closure, text) {
   for (const m of text.matchAll(/uses: \.\/(\S+)/g)) closure.add(`${m[1]}/action.yml`);
   for (const m of text.matchAll(/\b(?:bash|sh|node|bun) ((?:scripts|packages)\/\S+)/g))
-    closure.add(m[1], true);
+    seedRun(closure, m[1]);
   for (const m of text.matchAll(/\bbun run --cwd (\S+) ([\w:-]+)/g)) closure.script(m[1], m[2]);
 }
 
@@ -196,7 +203,12 @@ const cliCommands = (texts) =>
 
 function benchInputs() {
   const closure = new Closure();
-  const steps = ["build", "studio-edit-accuracy", "studio-edit-accuracy-gate"].map(job);
+  const steps = [
+    "build",
+    "studio-edit-accuracy",
+    "studio-edit-accuracy-retry",
+    "studio-edit-accuracy-gate",
+  ].map(job);
   steps.forEach((text) => seed(closure, text));
   for (const name of cliCommands([...steps, read(`${BENCH}/case.mjs`)]))
     closure.add(`packages/cli/src/commands/${name}.ts`);

scripts/ci/edit-accuracy-shard.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+# edit-accuracy-shard.sh <i/n>: runs one edit accuracy shard into /tmp/edit-accuracy, then re-runs twice each case
+# whose verdict differs from the base branch (/tmp/base-edit-accuracy.json). A slow runner makes the bench exit 75.
+set -euo pipefail
+bench() { bun run --cwd packages/studio test:edit-accuracy -- --grid "$EDIT_ACCURACY_GRID" --jobs 2 "$@"; }
+bench --shard "$1" --out /tmp/edit-accuracy/run1
+mapfile -t FLIPPED < <(node packages/studio/tests/e2e/edit-accuracy/ratchet.mjs flipped \
+  /tmp/base-edit-accuracy.json /tmp/edit-accuracy/run1/results.json)
+if (( ${#FLIPPED[@]} > 0 )); then
+  FILTER="^($(IFS='|'; echo "${FLIPPED[*]}"))$"
+  for run in 2 3; do bench --filter "$FILTER" --rerun --out "/tmp/edit-accuracy/run${run}"; done
+fi

scripts/edit-accuracy-retry.test.mjs

@@ -0,0 +1,103 @@
+import assert from "node:assert/strict";
+import { execFileSync } from "node:child_process";
+import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import test from "node:test";
+import { parse } from "yaml";
+
+// A slow runner must not drop a PR from the merge queue: its shard is marked, re-run once on a fresh runner,
+// and the re-run's evidence is what the gate reads.
+const ci = parse(
+  readFileSync(join(import.meta.dirname, "..", ".github", "workflows", "ci.yml"), "utf8"),
+);
+const jobs = ci.jobs;
+const step = (job, name) => jobs[job].steps.find((s) => s.name === name);
+
+// Runs a step's shell with its fixed paths moved under root; returns what it wrote to GITHUB_OUTPUT.
+function runStep(script, root, env = {}) {
+  const output = join(root, "github-output");
+  writeFileSync(output, "");
+  execFileSync("bash", ["-euo", "pipefail", "-c", script.replaceAll("/tmp/", `${root}/`)], {
+    env: { ...process.env, GITHUB_OUTPUT: output, ...env },
+  });
+  return readFileSync(output, "utf8");
+}
+
+function withTree(fn) {
+  const root = mkdtempSync(join(tmpdir(), "edit-accuracy-retry-"));
+  try {
+    return fn(root);
+  } finally {
+    rmSync(root, { recursive: true, force: true });
+  }
+}
+
+const markScript = (status) =>
+  step("studio-edit-accuracy", "Mark a slow or cancelled shard for a re-run").run.replaceAll(
+    "${{ job.status }}",
+    status,
+  );
+
+test("a shard is marked for a re-run when it timed out or its bench stopped on a slow runner", () => {
+  withTree((root) => {
+    runStep(markScript("cancelled"), root);
+    assert.ok(existsSync(join(root, "edit-accuracy/retry")));
+  });
+  withTree((root) => {
+    mkdirSync(join(root, "edit-accuracy/run1"), { recursive: true });
+    writeFileSync(join(root, "edit-accuracy/run1/slow-runner.json"), "{}");
+    runStep(markScript("failure"), root);
+    assert.ok(existsSync(join(root, "edit-accuracy/retry")));
+  });
+  withTree((root) => {
+    runStep(markScript("success"), root);
+    assert.ok(!existsSync(join(root, "edit-accuracy/retry")));
+  });
+});
+
+test("the retry plan lists exactly the marked shards", () => {
+  const plan = step("studio-edit-accuracy-retry-plan", "List the shards marked for a re-run").run;
+  withTree((root) => {
+    for (const shard of [7, 12, 3])
+      mkdirSync(join(root, `edit-accuracy-shards/edit-accuracy-shard-${shard}`), {
+        recursive: true,
+      });
+    writeFileSync(join(root, "edit-accuracy-shards/edit-accuracy-shard-7/retry"), "");
+    writeFileSync(join(root, "edit-accuracy-shards/edit-accuracy-shard-12/retry"), "");
+    assert.equal(runStep(plan, root), "shards=[7,12]\n");
+  });
+  withTree((root) => {
+    mkdirSync(join(root, "edit-accuracy-shards/edit-accuracy-shard-1"), { recursive: true });
+    assert.equal(runStep(plan, root), "shards=[]\n");
+  });
+});
+
+test("each marked shard re-runs once and replaces its evidence before the gate reads it", () => {
+  const retry = jobs["studio-edit-accuracy-retry"];
+  assert.match(retry.needs.join(" "), /studio-edit-accuracy-retry-plan/);
+  assert.match(
+    retry.strategy.matrix.shard,
+    /needs\.studio-edit-accuracy-retry-plan\.outputs\.shards/,
+  );
+  const shardRun = retry.steps.find((s) => s.run?.includes("edit-accuracy-shard.sh"));
+  assert.ok(shardRun, "the retry runs the same shard script as the shard job");
+  assert.ok(
+    jobs["studio-edit-accuracy"].steps.some((s) =>
+      s.run?.includes("scripts/ci/edit-accuracy-shard.sh"),
+    ),
+  );
+  const upload = retry.steps.find((s) => s.uses?.startsWith("actions/upload-artifact@"));
+  assert.equal(upload.with.name, "edit-accuracy-shard-${{ matrix.shard }}");
+  assert.equal(upload.with.overwrite, true);
+  assert.ok(jobs["studio-edit-accuracy-gate"].needs.includes("studio-edit-accuracy-retry"));
+  assert.match(jobs["studio-edit-accuracy-gate"].if, /!cancelled\(\)/);
+});
+
+test("a cancelled shard and its re-run both fit in the merge queue's 60 minute check wait", () => {
+  const shard = jobs["studio-edit-accuracy"]["timeout-minutes"];
+  const plan = jobs["studio-edit-accuracy-retry-plan"]["timeout-minutes"];
+  const retry = jobs["studio-edit-accuracy-retry"]["timeout-minutes"];
+  const gate = jobs["studio-edit-accuracy-gate"]["timeout-minutes"];
+  assert.ok(shard + plan + retry + gate <= 60, `${shard}+${plan}+${retry}+${gate} min`);
+});