From 9f63598e6b2b4096b1ca98c93285fc2e79167f17 Mon Sep 17 00:00:00 2001 From: Miguel Angel Simon Sierra Date: Sat, 1 Aug 2026 23:21:55 +0200 Subject: [PATCH] feat(evals): name the operating point at the top of the round 4 report The page carried every number and no answer: its headline was the sweep's 'ship nothing yet', which is what the rule says when the arm it ships depends on the list length, and the reader had to derive the point from twenty-two pairing tables. Read the family off the band and the list length off the adjacent-k pairings, and say in the same breath that the rule fixed the first and not the second. --- evals/catalog-retrieval/round2/harness2.py | 14 +++ .../catalog-retrieval/round2/test_harness2.py | 33 ++++++ .../catalog-retrieval/round4/build_report4.py | 99 ++++++++++++++++- .../round4/test_build_report4.py | 105 ++++++++++++++++++ 4 files changed, 250 insertions(+), 1 deletion(-) diff --git a/evals/catalog-retrieval/round2/harness2.py b/evals/catalog-retrieval/round2/harness2.py index 8aca1eee9d..3c722a7502 100644 --- a/evals/catalog-retrieval/round2/harness2.py +++ b/evals/catalog-retrieval/round2/harness2.py @@ -172,6 +172,20 @@ def hybrid_topk(brief_id, brief, entries, k): return [name for _, name in fused[:k]] +# Which retriever family a condition letter is. The sweep names arms by family +# and the paid grid names cells by condition, so anything that reads both has to +# cross the two, and it does it here rather than carrying its own idea of what +# "h" means. Kept beside build_context, whose branches it must cover: the a and b +# controls retrieve nothing and everything, so they name no family. +FAMILY_BY_CONDITION = { + "a": None, + "b": None, + "c": "lexical", + "d": "semantic", + "h": "hybrid", +} + + def build_context(condition, brief_id, brief, entries, k): """Return a context and the entry names shown in its one catalog slot.""" if condition == "a": diff --git a/evals/catalog-retrieval/round2/test_harness2.py b/evals/catalog-retrieval/round2/test_harness2.py index e1b1727041..6aa3c5cf72 100644 --- a/evals/catalog-retrieval/round2/test_harness2.py +++ b/evals/catalog-retrieval/round2/test_harness2.py @@ -3,10 +3,12 @@ import contextlib import importlib.util +import inspect import io import json import math import os +import re import tempfile import unittest import warnings @@ -264,6 +266,37 @@ class Harness2Tests(unittest.TestCase): self.assertEqual(runner, os.path.realpath(validate_gold.BRIEFS_DIR)) +class FamilyByConditionTests(unittest.TestCase): + """The letter to family map has to cover every condition build_context takes. + + Anything reading both the sweep and the paid grid crosses those two + vocabularies through this map, and a condition missing from it reads as a + family of None, which silently drops that cell from whatever was being + decided rather than failing. + """ + + def conditions_build_context_accepts(self): + source = inspect.getsource(harness2.build_context) + return set(re.findall(r"condition == \"(\w+)\"", source)) + + def test_every_condition_the_dispatch_takes_has_a_family(self): + taken = self.conditions_build_context_accepts() + self.assertTrue(taken, "no condition branches found, the regex went stale") + self.assertEqual(taken, set(harness2.FAMILY_BY_CONDITION)) + + def test_the_two_controls_name_no_family(self): + # a retrieves nothing and b retrieves everything, so neither is a + # retriever whose family could win anything. + self.assertIsNone(harness2.FAMILY_BY_CONDITION["a"]) + self.assertIsNone(harness2.FAMILY_BY_CONDITION["b"]) + + def test_the_three_retrievers_name_the_families_the_sweep_uses(self): + self.assertEqual( + {"c": "lexical", "d": "semantic", "h": "hybrid"}, + {k: v for k, v in harness2.FAMILY_BY_CONDITION.items() if v}, + ) + + class Embed2Tests(unittest.TestCase): def setUp(self): self.transport_guard = mock.patch.object( diff --git a/evals/catalog-retrieval/round4/build_report4.py b/evals/catalog-retrieval/round4/build_report4.py index f32adc3023..65bcc266e6 100644 --- a/evals/catalog-retrieval/round4/build_report4.py +++ b/evals/catalog-retrieval/round4/build_report4.py @@ -28,6 +28,7 @@ ROUND3 = os.path.join(os.path.dirname(HERE), "round3") sys.path.insert(0, ROUND3) import build_report3 # noqa: E402 +import confirm3 # noqa: E402 import recall # noqa: E402 import stats # noqa: E402 import sweep # noqa: E402 @@ -143,11 +144,107 @@ def band_sections(summary): return sections +def _separates(interval): + return interval["lo"] > 0 or interval["hi"] < 0 + + +def operating_point(summary, confirmation): + """The shipped (arm, k), read off the band and the adjacent-k pairings. + + The rule pre-registered how to pick the arm and it did not pick a k: its + outcome here is `k-dependent`, which says in as many words that the open + question is what k rather than which arm. So the k is chosen after the + numbers existed, by the only mechanical reading that answers "is a longer + list still buying anything": the largest bought list length whose step up + from the one below it separates on pass rate on at least one model. + + Computed rather than written down so it cannot drift from the confirmation + it claims to summarise, and labelled post hoc wherever it is rendered, + because a number the rule did not pre-register must not be presented as one + the rule decided. + """ + band = summary.get("band") + if not confirmation or not band or not band.get("clears"): + return None + family = band["band"]["family"] + steps = {} + for pair in confirmation.get("paired", []): + if tuple(pair.get("changed") or ()) != ("k",): + continue + cell = confirm3.parse_cell(pair["cell"]) + against = confirm3.parse_cell(pair["against"]) + if harness2.FAMILY_BY_CONDITION.get(cell["arm"]) != family: + continue + steps.setdefault((against["k"], cell["k"]), []).append( + _separates(pair["pass_rate"]) + ) + if not steps: + return None + climbing = sorted(step for step, votes in steps.items() if any(votes)) + if not climbing: + return None + return { + "family": family, + "k": climbing[-1][1], + "from_k": climbing[-1][0], + "models": len(steps[climbing[-1]]), + "stalled": sorted(step for step, votes in steps.items() if not any(votes)), + } + + +def operating_point_sections(summary, confirmation): + """The one thing a reader opens this page for, before the six tables.""" + point = operating_point(summary, confirmation) + if point is None: + return [ + build_report3._paragraph( + "Operating point: none yet. Either the band did not clear, or no " + "confirmation has been bought at two list lengths of the leading " + "family, so nothing on this page answers what k to ship." + ) + ] + stalled = ", ".join( + "{0} to {1}".format(lo, hi) for lo, hi in point["stalled"] if lo >= point["k"] + ) + return [ + build_report3._paragraph( + "Operating point: {0} at k={1}. The family is what the band decided, " + "over {2} contiguous list lengths against a required {3}. The list " + "length is the last one that bought anything: going from k={4} to " + "k={1} separated on pass rate on {5} of the models confirmed there, " + "and {6}.".format( + point["family"], + point["k"], + summary["band"]["width"], + summary["band"]["required"], + point["from_k"], + point["models"], + "every longer step measured here separated on none of them ({0})".format( + stalled + ) + if stalled + else "no longer list length was bought, so this is the longest " + "measured rather than a measured ceiling", + ) + ), + build_report3._paragraph( + "The list length above was chosen after the numbers existed. The rule " + "fixed how to pick the arm and required confirmation at several list " + "lengths; it did not fix how to pick k inside the band, and its own " + "outcome on this corpus is k-dependent. Read it as the reading of the " + "confirmation that this page can defend, not as something the " + "pre-registered rule decided." + ), + ] + + def render(summary, confirmation=None): return build_report3.render( summary, confirmation, - leading_sections=band_sections(summary), + leading_sections=( + operating_point_sections(summary, confirmation) + band_sections(summary) + ), title=TITLE, ) diff --git a/evals/catalog-retrieval/round4/test_build_report4.py b/evals/catalog-retrieval/round4/test_build_report4.py index 6c6b03a7cc..5a6c4a8d78 100644 --- a/evals/catalog-retrieval/round4/test_build_report4.py +++ b/evals/catalog-retrieval/round4/test_build_report4.py @@ -218,6 +218,111 @@ class ReportBandProseTests(unittest.TestCase): self.assertNotIn("Band stability", page) +def step(arm, lo_k, hi_k, model, separates, metric="pass_rate"): + """One adjacent-list-length pairing, shaped like confirm3.paired's output.""" + flat = {"mean": 0.0, "lo": -0.02, "hi": 0.02, "se": 0.01, "clusters": 9, "n": 9} + up = {"mean": 0.05, "lo": 0.01, "hi": 0.09, "se": 0.02, "clusters": 9, "n": 9} + out = { + "cell": "{0}@{1}/{2}".format(arm, hi_k, model), + "against": "{0}@{1}/{2}".format(arm, lo_k, model), + "changed": ["k"], + "n": 9, + } + for name, _ in build_report4.confirm3.METRICS: + out[name] = dict(up if (separates and name == metric) else flat) + return out + + +class OperatingPointTests(unittest.TestCase): + """The page has to name what ships, and say who decided the list length. + + The band decides the family and the rule stops there: its own outcome on + this corpus is k-dependent, so the k comes off the confirmation instead and + has to be labelled as chosen after the numbers existed. + """ + + def summary(self): + return summarise(ONE_BAND) + + def confirmation(self, paired): + return {"cells": {}, "paired": paired} + + def test_the_shipped_k_is_the_last_step_that_bought_anything(self): + point = build_report4.operating_point( + self.summary(), + self.confirmation( + [ + step("d", 10, 20, "model-a", True), + step("d", 20, 30, "model-a", True), + step("d", 30, 40, "model-a", False), + ] + ), + ) + self.assertEqual(("semantic", 30, 20), (point["family"], point["k"], point["from_k"])) + self.assertEqual([(30, 40)], point["stalled"]) + + def test_a_step_that_separates_on_one_model_of_two_still_counts(self): + # The claim being made is that a longer list still buys something, and + # one model measuring that is enough to make the claim. Requiring both + # would silently ship the shorter list whenever the models disagree. + point = build_report4.operating_point( + self.summary(), + self.confirmation( + [ + step("d", 10, 20, "model-a", True), + step("d", 20, 30, "model-a", False), + step("d", 20, 30, "model-b", True), + ] + ), + ) + self.assertEqual(30, point["k"]) + self.assertEqual(2, point["models"]) + + def test_another_family_cannot_choose_the_list_length(self): + # The band already decided the family. A lexical step still climbing at + # k=40 says nothing about how long the shipped arm's list should be. + point = build_report4.operating_point( + self.summary(), + self.confirmation( + [ + step("d", 10, 20, "model-a", True), + step("c", 20, 40, "model-a", True), + ] + ), + ) + self.assertEqual(20, point["k"]) + + def test_no_confirmation_ships_no_list_length(self): + self.assertIsNone(build_report4.operating_point(self.summary(), None)) + self.assertIsNone(build_report4.operating_point(self.summary(), {"paired": []})) + page = build_report4.render(self.summary()) + self.assertIn("Operating point: none yet", text_of(page)) + + def test_a_band_that_does_not_clear_ships_nothing_whatever_the_pairings_say(self): + summary = summarise(NEVER_SEPARATES) + self.assertIsNone( + build_report4.operating_point( + summary, self.confirmation([step("d", 10, 20, "model-a", True)]) + ) + ) + + def test_the_page_names_the_point_and_says_who_chose_the_list_length(self): + page = build_report4.render( + self.summary(), + self.confirmation( + [ + step("d", 10, 20, "model-a", True), + step("d", 20, 30, "model-a", False), + ] + ), + ) + text = text_of(page) + self.assertIn("Operating point: semantic at k=20", text) + self.assertIn("chosen after the numbers existed", text) + # And it sits in front of the band table it summarises, not after it. + self.assertLess(text.index("Operating point"), text.index("Band stability")) + + class NoEmDashTests(unittest.TestCase): """The count has to be zero, and spelling the character out would fail this file."""