feat(evals): name the operating point at the top of the round 4 report

The page carried every number and no answer: its headline was the sweep's
'ship nothing yet', which is what the rule says when the arm it ships depends
on the list length, and the reader had to derive the point from twenty-two
pairing tables. Read the family off the band and the list length off the
adjacent-k pairings, and say in the same breath that the rule fixed the first
and not the second.
This commit is contained in:
Miguel Angel Simon Sierra
2026-08-01 23:21:55 +02:00
parent 779ce59225
commit 9f63598e6b
4 changed files with 250 additions and 1 deletions
@@ -172,6 +172,20 @@ def hybrid_topk(brief_id, brief, entries, k):
return [name for _, name in fused[:k]]
# Which retriever family a condition letter is. The sweep names arms by family
# and the paid grid names cells by condition, so anything that reads both has to
# cross the two, and it does it here rather than carrying its own idea of what
# "h" means. Kept beside build_context, whose branches it must cover: the a and b
# controls retrieve nothing and everything, so they name no family.
FAMILY_BY_CONDITION = {
"a": None,
"b": None,
"c": "lexical",
"d": "semantic",
"h": "hybrid",
}
def build_context(condition, brief_id, brief, entries, k):
"""Return a context and the entry names shown in its one catalog slot."""
if condition == "a":
@@ -3,10 +3,12 @@
import contextlib
import importlib.util
import inspect
import io
import json
import math
import os
import re
import tempfile
import unittest
import warnings
@@ -264,6 +266,37 @@ class Harness2Tests(unittest.TestCase):
self.assertEqual(runner, os.path.realpath(validate_gold.BRIEFS_DIR))
class FamilyByConditionTests(unittest.TestCase):
"""The letter to family map has to cover every condition build_context takes.
Anything reading both the sweep and the paid grid crosses those two
vocabularies through this map, and a condition missing from it reads as a
family of None, which silently drops that cell from whatever was being
decided rather than failing.
"""
def conditions_build_context_accepts(self):
source = inspect.getsource(harness2.build_context)
return set(re.findall(r"condition == \"(\w+)\"", source))
def test_every_condition_the_dispatch_takes_has_a_family(self):
taken = self.conditions_build_context_accepts()
self.assertTrue(taken, "no condition branches found, the regex went stale")
self.assertEqual(taken, set(harness2.FAMILY_BY_CONDITION))
def test_the_two_controls_name_no_family(self):
# a retrieves nothing and b retrieves everything, so neither is a
# retriever whose family could win anything.
self.assertIsNone(harness2.FAMILY_BY_CONDITION["a"])
self.assertIsNone(harness2.FAMILY_BY_CONDITION["b"])
def test_the_three_retrievers_name_the_families_the_sweep_uses(self):
self.assertEqual(
{"c": "lexical", "d": "semantic", "h": "hybrid"},
{k: v for k, v in harness2.FAMILY_BY_CONDITION.items() if v},
)
class Embed2Tests(unittest.TestCase):
def setUp(self):
self.transport_guard = mock.patch.object(
@@ -28,6 +28,7 @@ ROUND3 = os.path.join(os.path.dirname(HERE), "round3")
sys.path.insert(0, ROUND3)
import build_report3 # noqa: E402
import confirm3 # noqa: E402
import recall # noqa: E402
import stats # noqa: E402
import sweep # noqa: E402
@@ -143,11 +144,107 @@ def band_sections(summary):
return sections
def _separates(interval):
return interval["lo"] > 0 or interval["hi"] < 0
def operating_point(summary, confirmation):
"""The shipped (arm, k), read off the band and the adjacent-k pairings.
The rule pre-registered how to pick the arm and it did not pick a k: its
outcome here is `k-dependent`, which says in as many words that the open
question is what k rather than which arm. So the k is chosen after the
numbers existed, by the only mechanical reading that answers "is a longer
list still buying anything": the largest bought list length whose step up
from the one below it separates on pass rate on at least one model.
Computed rather than written down so it cannot drift from the confirmation
it claims to summarise, and labelled post hoc wherever it is rendered,
because a number the rule did not pre-register must not be presented as one
the rule decided.
"""
band = summary.get("band")
if not confirmation or not band or not band.get("clears"):
return None
family = band["band"]["family"]
steps = {}
for pair in confirmation.get("paired", []):
if tuple(pair.get("changed") or ()) != ("k",):
continue
cell = confirm3.parse_cell(pair["cell"])
against = confirm3.parse_cell(pair["against"])
if harness2.FAMILY_BY_CONDITION.get(cell["arm"]) != family:
continue
steps.setdefault((against["k"], cell["k"]), []).append(
_separates(pair["pass_rate"])
)
if not steps:
return None
climbing = sorted(step for step, votes in steps.items() if any(votes))
if not climbing:
return None
return {
"family": family,
"k": climbing[-1][1],
"from_k": climbing[-1][0],
"models": len(steps[climbing[-1]]),
"stalled": sorted(step for step, votes in steps.items() if not any(votes)),
}
def operating_point_sections(summary, confirmation):
"""The one thing a reader opens this page for, before the six tables."""
point = operating_point(summary, confirmation)
if point is None:
return [
build_report3._paragraph(
"Operating point: none yet. Either the band did not clear, or no "
"confirmation has been bought at two list lengths of the leading "
"family, so nothing on this page answers what k to ship."
)
]
stalled = ", ".join(
"{0} to {1}".format(lo, hi) for lo, hi in point["stalled"] if lo >= point["k"]
)
return [
build_report3._paragraph(
"Operating point: {0} at k={1}. The family is what the band decided, "
"over {2} contiguous list lengths against a required {3}. The list "
"length is the last one that bought anything: going from k={4} to "
"k={1} separated on pass rate on {5} of the models confirmed there, "
"and {6}.".format(
point["family"],
point["k"],
summary["band"]["width"],
summary["band"]["required"],
point["from_k"],
point["models"],
"every longer step measured here separated on none of them ({0})".format(
stalled
)
if stalled
else "no longer list length was bought, so this is the longest "
"measured rather than a measured ceiling",
)
),
build_report3._paragraph(
"The list length above was chosen after the numbers existed. The rule "
"fixed how to pick the arm and required confirmation at several list "
"lengths; it did not fix how to pick k inside the band, and its own "
"outcome on this corpus is k-dependent. Read it as the reading of the "
"confirmation that this page can defend, not as something the "
"pre-registered rule decided."
),
]
def render(summary, confirmation=None):
return build_report3.render(
summary,
confirmation,
leading_sections=band_sections(summary),
leading_sections=(
operating_point_sections(summary, confirmation) + band_sections(summary)
),
title=TITLE,
)
@@ -218,6 +218,111 @@ class ReportBandProseTests(unittest.TestCase):
self.assertNotIn("Band stability", page)
def step(arm, lo_k, hi_k, model, separates, metric="pass_rate"):
"""One adjacent-list-length pairing, shaped like confirm3.paired's output."""
flat = {"mean": 0.0, "lo": -0.02, "hi": 0.02, "se": 0.01, "clusters": 9, "n": 9}
up = {"mean": 0.05, "lo": 0.01, "hi": 0.09, "se": 0.02, "clusters": 9, "n": 9}
out = {
"cell": "{0}@{1}/{2}".format(arm, hi_k, model),
"against": "{0}@{1}/{2}".format(arm, lo_k, model),
"changed": ["k"],
"n": 9,
}
for name, _ in build_report4.confirm3.METRICS:
out[name] = dict(up if (separates and name == metric) else flat)
return out
class OperatingPointTests(unittest.TestCase):
"""The page has to name what ships, and say who decided the list length.
The band decides the family and the rule stops there: its own outcome on
this corpus is k-dependent, so the k comes off the confirmation instead and
has to be labelled as chosen after the numbers existed.
"""
def summary(self):
return summarise(ONE_BAND)
def confirmation(self, paired):
return {"cells": {}, "paired": paired}
def test_the_shipped_k_is_the_last_step_that_bought_anything(self):
point = build_report4.operating_point(
self.summary(),
self.confirmation(
[
step("d", 10, 20, "model-a", True),
step("d", 20, 30, "model-a", True),
step("d", 30, 40, "model-a", False),
]
),
)
self.assertEqual(("semantic", 30, 20), (point["family"], point["k"], point["from_k"]))
self.assertEqual([(30, 40)], point["stalled"])
def test_a_step_that_separates_on_one_model_of_two_still_counts(self):
# The claim being made is that a longer list still buys something, and
# one model measuring that is enough to make the claim. Requiring both
# would silently ship the shorter list whenever the models disagree.
point = build_report4.operating_point(
self.summary(),
self.confirmation(
[
step("d", 10, 20, "model-a", True),
step("d", 20, 30, "model-a", False),
step("d", 20, 30, "model-b", True),
]
),
)
self.assertEqual(30, point["k"])
self.assertEqual(2, point["models"])
def test_another_family_cannot_choose_the_list_length(self):
# The band already decided the family. A lexical step still climbing at
# k=40 says nothing about how long the shipped arm's list should be.
point = build_report4.operating_point(
self.summary(),
self.confirmation(
[
step("d", 10, 20, "model-a", True),
step("c", 20, 40, "model-a", True),
]
),
)
self.assertEqual(20, point["k"])
def test_no_confirmation_ships_no_list_length(self):
self.assertIsNone(build_report4.operating_point(self.summary(), None))
self.assertIsNone(build_report4.operating_point(self.summary(), {"paired": []}))
page = build_report4.render(self.summary())
self.assertIn("Operating point: none yet", text_of(page))
def test_a_band_that_does_not_clear_ships_nothing_whatever_the_pairings_say(self):
summary = summarise(NEVER_SEPARATES)
self.assertIsNone(
build_report4.operating_point(
summary, self.confirmation([step("d", 10, 20, "model-a", True)])
)
)
def test_the_page_names_the_point_and_says_who_chose_the_list_length(self):
page = build_report4.render(
self.summary(),
self.confirmation(
[
step("d", 10, 20, "model-a", True),
step("d", 20, 30, "model-a", False),
]
),
)
text = text_of(page)
self.assertIn("Operating point: semantic at k=20", text)
self.assertIn("chosen after the numbers existed", text)
# And it sits in front of the band table it summarises, not after it.
self.assertLess(text.index("Operating point"), text.index("Band stability"))
class NoEmDashTests(unittest.TestCase):
"""The count has to be zero, and spelling the character out would fail this file."""