diff --git a/evals/catalog-retrieval/round2/harness2.py b/evals/catalog-retrieval/round2/harness2.py index 8aca1eee9d..3c722a7502 100644 --- a/evals/catalog-retrieval/round2/harness2.py +++ b/evals/catalog-retrieval/round2/harness2.py @@ -172,6 +172,20 @@ def hybrid_topk(brief_id, brief, entries, k): return [name for _, name in fused[:k]] +# Which retriever family a condition letter is. The sweep names arms by family +# and the paid grid names cells by condition, so anything that reads both has to +# cross the two, and it does it here rather than carrying its own idea of what +# "h" means. Kept beside build_context, whose branches it must cover: the a and b +# controls retrieve nothing and everything, so they name no family. +FAMILY_BY_CONDITION = { + "a": None, + "b": None, + "c": "lexical", + "d": "semantic", + "h": "hybrid", +} + + def build_context(condition, brief_id, brief, entries, k): """Return a context and the entry names shown in its one catalog slot.""" if condition == "a": diff --git a/evals/catalog-retrieval/round2/test_harness2.py b/evals/catalog-retrieval/round2/test_harness2.py index e1b1727041..6aa3c5cf72 100644 --- a/evals/catalog-retrieval/round2/test_harness2.py +++ b/evals/catalog-retrieval/round2/test_harness2.py @@ -3,10 +3,12 @@ import contextlib import importlib.util +import inspect import io import json import math import os +import re import tempfile import unittest import warnings @@ -264,6 +266,37 @@ class Harness2Tests(unittest.TestCase): self.assertEqual(runner, os.path.realpath(validate_gold.BRIEFS_DIR)) +class FamilyByConditionTests(unittest.TestCase): + """The letter to family map has to cover every condition build_context takes. + + Anything reading both the sweep and the paid grid crosses those two + vocabularies through this map, and a condition missing from it reads as a + family of None, which silently drops that cell from whatever was being + decided rather than failing. + """ + + def conditions_build_context_accepts(self): + source = inspect.getsource(harness2.build_context) + return set(re.findall(r"condition == \"(\w+)\"", source)) + + def test_every_condition_the_dispatch_takes_has_a_family(self): + taken = self.conditions_build_context_accepts() + self.assertTrue(taken, "no condition branches found, the regex went stale") + self.assertEqual(taken, set(harness2.FAMILY_BY_CONDITION)) + + def test_the_two_controls_name_no_family(self): + # a retrieves nothing and b retrieves everything, so neither is a + # retriever whose family could win anything. + self.assertIsNone(harness2.FAMILY_BY_CONDITION["a"]) + self.assertIsNone(harness2.FAMILY_BY_CONDITION["b"]) + + def test_the_three_retrievers_name_the_families_the_sweep_uses(self): + self.assertEqual( + {"c": "lexical", "d": "semantic", "h": "hybrid"}, + {k: v for k, v in harness2.FAMILY_BY_CONDITION.items() if v}, + ) + + class Embed2Tests(unittest.TestCase): def setUp(self): self.transport_guard = mock.patch.object( diff --git a/evals/catalog-retrieval/round4/build_report4.py b/evals/catalog-retrieval/round4/build_report4.py index f32adc3023..65bcc266e6 100644 --- a/evals/catalog-retrieval/round4/build_report4.py +++ b/evals/catalog-retrieval/round4/build_report4.py @@ -28,6 +28,7 @@ ROUND3 = os.path.join(os.path.dirname(HERE), "round3") sys.path.insert(0, ROUND3) import build_report3 # noqa: E402 +import confirm3 # noqa: E402 import recall # noqa: E402 import stats # noqa: E402 import sweep # noqa: E402 @@ -143,11 +144,107 @@ def band_sections(summary): return sections +def _separates(interval): + return interval["lo"] > 0 or interval["hi"] < 0 + + +def operating_point(summary, confirmation): + """The shipped (arm, k), read off the band and the adjacent-k pairings. + + The rule pre-registered how to pick the arm and it did not pick a k: its + outcome here is `k-dependent`, which says in as many words that the open + question is what k rather than which arm. So the k is chosen after the + numbers existed, by the only mechanical reading that answers "is a longer + list still buying anything": the largest bought list length whose step up + from the one below it separates on pass rate on at least one model. + + Computed rather than written down so it cannot drift from the confirmation + it claims to summarise, and labelled post hoc wherever it is rendered, + because a number the rule did not pre-register must not be presented as one + the rule decided. + """ + band = summary.get("band") + if not confirmation or not band or not band.get("clears"): + return None + family = band["band"]["family"] + steps = {} + for pair in confirmation.get("paired", []): + if tuple(pair.get("changed") or ()) != ("k",): + continue + cell = confirm3.parse_cell(pair["cell"]) + against = confirm3.parse_cell(pair["against"]) + if harness2.FAMILY_BY_CONDITION.get(cell["arm"]) != family: + continue + steps.setdefault((against["k"], cell["k"]), []).append( + _separates(pair["pass_rate"]) + ) + if not steps: + return None + climbing = sorted(step for step, votes in steps.items() if any(votes)) + if not climbing: + return None + return { + "family": family, + "k": climbing[-1][1], + "from_k": climbing[-1][0], + "models": len(steps[climbing[-1]]), + "stalled": sorted(step for step, votes in steps.items() if not any(votes)), + } + + +def operating_point_sections(summary, confirmation): + """The one thing a reader opens this page for, before the six tables.""" + point = operating_point(summary, confirmation) + if point is None: + return [ + build_report3._paragraph( + "Operating point: none yet. Either the band did not clear, or no " + "confirmation has been bought at two list lengths of the leading " + "family, so nothing on this page answers what k to ship." + ) + ] + stalled = ", ".join( + "{0} to {1}".format(lo, hi) for lo, hi in point["stalled"] if lo >= point["k"] + ) + return [ + build_report3._paragraph( + "Operating point: {0} at k={1}. The family is what the band decided, " + "over {2} contiguous list lengths against a required {3}. The list " + "length is the last one that bought anything: going from k={4} to " + "k={1} separated on pass rate on {5} of the models confirmed there, " + "and {6}.".format( + point["family"], + point["k"], + summary["band"]["width"], + summary["band"]["required"], + point["from_k"], + point["models"], + "every longer step measured here separated on none of them ({0})".format( + stalled + ) + if stalled + else "no longer list length was bought, so this is the longest " + "measured rather than a measured ceiling", + ) + ), + build_report3._paragraph( + "The list length above was chosen after the numbers existed. The rule " + "fixed how to pick the arm and required confirmation at several list " + "lengths; it did not fix how to pick k inside the band, and its own " + "outcome on this corpus is k-dependent. Read it as the reading of the " + "confirmation that this page can defend, not as something the " + "pre-registered rule decided." + ), + ] + + def render(summary, confirmation=None): return build_report3.render( summary, confirmation, - leading_sections=band_sections(summary), + leading_sections=( + operating_point_sections(summary, confirmation) + band_sections(summary) + ), title=TITLE, ) diff --git a/evals/catalog-retrieval/round4/test_build_report4.py b/evals/catalog-retrieval/round4/test_build_report4.py index 6c6b03a7cc..5a6c4a8d78 100644 --- a/evals/catalog-retrieval/round4/test_build_report4.py +++ b/evals/catalog-retrieval/round4/test_build_report4.py @@ -218,6 +218,111 @@ class ReportBandProseTests(unittest.TestCase): self.assertNotIn("Band stability", page) +def step(arm, lo_k, hi_k, model, separates, metric="pass_rate"): + """One adjacent-list-length pairing, shaped like confirm3.paired's output.""" + flat = {"mean": 0.0, "lo": -0.02, "hi": 0.02, "se": 0.01, "clusters": 9, "n": 9} + up = {"mean": 0.05, "lo": 0.01, "hi": 0.09, "se": 0.02, "clusters": 9, "n": 9} + out = { + "cell": "{0}@{1}/{2}".format(arm, hi_k, model), + "against": "{0}@{1}/{2}".format(arm, lo_k, model), + "changed": ["k"], + "n": 9, + } + for name, _ in build_report4.confirm3.METRICS: + out[name] = dict(up if (separates and name == metric) else flat) + return out + + +class OperatingPointTests(unittest.TestCase): + """The page has to name what ships, and say who decided the list length. + + The band decides the family and the rule stops there: its own outcome on + this corpus is k-dependent, so the k comes off the confirmation instead and + has to be labelled as chosen after the numbers existed. + """ + + def summary(self): + return summarise(ONE_BAND) + + def confirmation(self, paired): + return {"cells": {}, "paired": paired} + + def test_the_shipped_k_is_the_last_step_that_bought_anything(self): + point = build_report4.operating_point( + self.summary(), + self.confirmation( + [ + step("d", 10, 20, "model-a", True), + step("d", 20, 30, "model-a", True), + step("d", 30, 40, "model-a", False), + ] + ), + ) + self.assertEqual(("semantic", 30, 20), (point["family"], point["k"], point["from_k"])) + self.assertEqual([(30, 40)], point["stalled"]) + + def test_a_step_that_separates_on_one_model_of_two_still_counts(self): + # The claim being made is that a longer list still buys something, and + # one model measuring that is enough to make the claim. Requiring both + # would silently ship the shorter list whenever the models disagree. + point = build_report4.operating_point( + self.summary(), + self.confirmation( + [ + step("d", 10, 20, "model-a", True), + step("d", 20, 30, "model-a", False), + step("d", 20, 30, "model-b", True), + ] + ), + ) + self.assertEqual(30, point["k"]) + self.assertEqual(2, point["models"]) + + def test_another_family_cannot_choose_the_list_length(self): + # The band already decided the family. A lexical step still climbing at + # k=40 says nothing about how long the shipped arm's list should be. + point = build_report4.operating_point( + self.summary(), + self.confirmation( + [ + step("d", 10, 20, "model-a", True), + step("c", 20, 40, "model-a", True), + ] + ), + ) + self.assertEqual(20, point["k"]) + + def test_no_confirmation_ships_no_list_length(self): + self.assertIsNone(build_report4.operating_point(self.summary(), None)) + self.assertIsNone(build_report4.operating_point(self.summary(), {"paired": []})) + page = build_report4.render(self.summary()) + self.assertIn("Operating point: none yet", text_of(page)) + + def test_a_band_that_does_not_clear_ships_nothing_whatever_the_pairings_say(self): + summary = summarise(NEVER_SEPARATES) + self.assertIsNone( + build_report4.operating_point( + summary, self.confirmation([step("d", 10, 20, "model-a", True)]) + ) + ) + + def test_the_page_names_the_point_and_says_who_chose_the_list_length(self): + page = build_report4.render( + self.summary(), + self.confirmation( + [ + step("d", 10, 20, "model-a", True), + step("d", 20, 30, "model-a", False), + ] + ), + ) + text = text_of(page) + self.assertIn("Operating point: semantic at k=20", text) + self.assertIn("chosen after the numbers existed", text) + # And it sits in front of the band table it summarises, not after it. + self.assertLess(text.index("Operating point"), text.index("Band stability")) + + class NoEmDashTests(unittest.TestCase): """The count has to be zero, and spelling the character out would fail this file."""