fix(studio): the edit-accuracy gate says the shards did not run, not that accuracy fell (#6069)

With no shard results, ratchet.mjs threw, wrote no report, and the workflow
printed "Edit accuracy fell against the base branch" for what was a build or
shard failure. A shard that did not report was also counted as a drop. The
gate now exits 2 with "Edit accuracy did not run: N of M shards reported
results" when nothing that ran fell, writes its comment and report in that
case too, and the workflow prints the gate's own verdict.

.github/workflows/ci.yml

@@ -1310,7 +1310,8 @@ jobs:
       - name: Fail if the gate failed
         if: steps.gate.outputs.exit_code != '0'
         run: |
-          echo "::error::Edit accuracy fell against the base branch; see the PR comment."
+          verdict=$(jq -r '.verdict // empty' /tmp/edit-accuracy-gate/gate.json 2>/dev/null)
+          echo "::error::${verdict:-The edit accuracy gate could not finish; see its log.}"
           exit 1
 
   smoke-global-install:

packages/studio/tests/e2e/edit-accuracy/ratchet.mjs

@@ -157,6 +157,20 @@ const readJson = (path) => JSON.parse(readFileSync(path, "utf8"));
 /** The grid a shard ran, from its first run's meta ("pr shard 2/5"); re-runs record their filter instead. */
 const gridOf = (results) => results.find((r) => !r.meta.rerun)?.meta.grid.split(" ")[0];
 
+const VERDICTS = {
+  0: "Edit accuracy holds against the base branch.",
+  1: "Edit accuracy fell against the base branch; see the PR comment.",
+};
+const notRun = (reported) => `Edit accuracy did not run: ${reported}; see the shard jobs.`;
+
+/** Writes what the workflow posts and prints; the verdict is the only text it shows for a failed gate. */
+function report(out, markdown, g) {
+  mkdirSync(out, { recursive: true });
+  writeFileSync(join(out, "comment.md"), markdown);
+  writeFileSync(join(out, "gate.json"), JSON.stringify(g, null, 1));
+  console.log(markdown);
+}
+
 // fallow-ignore-next-line complexity
 function main(argv) {
   const shardsAt = argv.indexOf("--shards");
@@ -169,7 +183,18 @@ function main(argv) {
   }
   const [headPath, out, ...resultPaths] = rest;
   const runs = resultPaths.map((p) => readJson(p));
-  if (!runs.length) throw new Error("no results.json from any shard");
+  if (!runs.length) {
+    const verdict = notRun(`0 of ${shards ?? "the"} shards reported results`);
+    report(
+      out,
+      `<!-- edit-accuracy -->\n### ${verdict}\n\nNothing was compared with the base branch.\n`,
+      {
+        ok: false,
+        verdict,
+      },
+    );
+    return 2;
+  }
   const cases = runs.flatMap((r) => r.cases);
   const full = gridOf(runs) === "full";
   const g = gate(
@@ -181,21 +206,21 @@ function main(argv) {
   );
   // A shard that died uploads nothing, so the count of first runs is what shows it.
   const firstRuns = runs.filter((r) => !r.meta.rerun).length;
-  if (shards !== null && firstRuns !== shards) {
-    g.reasons.push(`${firstRuns} of ${shards} shards reported results`);
-    g.ok = false;
-  }
-  mkdirSync(out, { recursive: true });
-  writeFileSync(join(out, "comment.md"), comment(g));
-  writeFileSync(join(out, "gate.json"), JSON.stringify(g, null, 1));
+  const shortfall =
+    shards !== null && firstRuns !== shards && `${firstRuns} of ${shards} shards reported results`;
+  // A case that ran and fell is a drop; a shard that did not report is not one.
+  const code = g.ok ? (shortfall ? 2 : 0) : 1;
+  if (shortfall) g.reasons.push(shortfall);
+  g.ok = code === 0;
+  g.verdict = code === 2 ? notRun(shortfall) : VERDICTS[code];
+  report(out, comment(g), g);
   // A baseline.json to commit when cases newly pass; only a full grid holds every case.
   if (full) writeReport(out, { ...runs[0].meta, grid: "full (CI)" }, bankable(cases), 0);
-  console.log(comment(g));
   if (g.unbanked.length && process.env.GITHUB_ACTIONS)
     console.log(
       `::notice title=Edit accuracy::${g.unbanked.length} newly passing case(s) are not banked in baseline.json; not gated.`,
     );
-  return g.ok ? 0 : 1;
+  return code;
 }
 
 if (process.argv[1] === fileURLToPath(import.meta.url))

packages/studio/tests/e2e/edit-accuracy/ratchet.test.mjs

@@ -1,3 +1,8 @@
+import { spawnSync } from "node:child_process";
+import { mkdtempSync, readFileSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { fileURLToPath } from "node:url";
 import { describe, expect, it } from "vitest";
 import { accurate, bankable, comment, flipped, gate, QUARANTINED } from "./ratchet.mjs";
 import { entry, score } from "./report.mjs";
@@ -190,3 +195,63 @@ describe("teleport in the gate", () => {
     expect(accurate(entry(score({ id: "nudge-x" }, { ...r, teleport: null })))).toBe(true);
   });
 });
+
+describe("gate command, as CI runs it", () => {
+  const script = fileURLToPath(new URL("./ratchet.mjs", import.meta.url));
+  /** Runs `gate --shards <shards>` over the given shard results; returns its exit code, comment and verdict. */
+  function runGate(shards, shardCases) {
+    const dir = mkdtempSync(join(tmpdir(), "edit-accuracy-gate-"));
+    const base = join(dir, "base.json");
+    writeFileSync(base, JSON.stringify(baseline({ a: good, b: good })));
+    const results = shardCases.map((cases, i) => {
+      const path = join(dir, `shard-${i}.json`);
+      writeFileSync(path, JSON.stringify({ meta: { grid: `pr shard ${i + 1}/${shards}` }, cases }));
+      return path;
+    });
+    const out = join(dir, "out");
+    const run = spawnSync(
+      process.execPath,
+      [script, "gate", "--shards", String(shards), base, base, out, ...results],
+      {
+        encoding: "utf8",
+      },
+    );
+    const report = JSON.parse(readFileSync(join(out, "gate.json"), "utf8"));
+    return {
+      code: run.status,
+      comment: readFileSync(join(out, "comment.md"), "utf8"),
+      verdict: report.verdict,
+    };
+  }
+
+  it("says the shards did not run when none reported results, never that accuracy fell", () => {
+    const g = runGate(5, []);
+    expect(g.code).toBe(2);
+    expect(g.verdict).toBe(
+      "Edit accuracy did not run: 0 of 5 shards reported results; see the shard jobs.",
+    );
+    expect(g.comment).toContain("0 of 5 shards reported results");
+    expect(g.comment).not.toMatch(/fell|accurate \d/);
+  });
+
+  it("says the shards did not all run when the cases that ran hold, so a missing shard is not a drop", () => {
+    const g = runGate(2, [[run("a")]]);
+    expect(g.code).toBe(2);
+    expect(g.verdict).toBe(
+      "Edit accuracy did not run: 1 of 2 shards reported results; see the shard jobs.",
+    );
+  });
+
+  it("still says accuracy fell when a case that ran regressed, even with a shard missing", () => {
+    const g = runGate(2, [[run("a", 9), run("a", 9), run("a", 9)]]);
+    expect(g.code).toBe(1);
+    expect(g.verdict).toBe("Edit accuracy fell against the base branch; see the PR comment.");
+    expect(g.comment).toContain("1 of 2 shards reported results");
+  });
+
+  it("passes when every shard reported and nothing fell", () => {
+    const g = runGate(2, [[run("a")], [run("b")]]);
+    expect(g.code).toBe(0);
+    expect(g.verdict).toBe("Edit accuracy holds against the base branch.");
+  });
+});