From a4e879f5ede2c77b5e69d20a7f2270aaf3a7a4ef Mon Sep 17 00:00:00 2001 From: Miguel Angel Simon Sierra Date: Fri, 31 Jul 2026 22:10:15 +0200 Subject: [PATCH] feat(eval): report round 3 with intervals and a named stop reason Round 3's arm ranking is decided by recall, which the retriever computes before a token is bought, so the report that carries it should cost nothing and be readable without the paid stage having run. Three things drove the shape. Every table ships a clustered interval from stats.py rather than a bare mean, because a recall number without one cannot be read against the pre-registered rule; that forced the recall curve into long form, since a wide arm-by-k grid has no room for error bars. The report names which of the three stopping conditions fired, because an arm that won on measurement, a round that ran out of corpus, and a round that ran out of budget are three different verdicts and collapsing them into one is how a tie gets reported as a win. And the summary file replaces infinite bounds with null, since JSON has no infinity and a file only one language can read back is not an auditable artifact. VERDICT.md is deliberately pending. The paid confirmation has not run, so it states the rule, the three stop reasons and the 424-entry shelf ceiling, and lists every absent number beside the file that will produce it, rather than estimating any of them. The ceiling is the load-bearing part: no corpus size buys more than 424 independent things to be right about, so if the arms do not separate the honest recommendation is a larger catalog, not a larger corpus. Tests are unittest, matching the rest of round 3, and parse the rendered HTML rather than grepping it, since a grep passes on a caption that merely mentions an interval. --- evals/catalog-retrieval/round3/VERDICT.md | 128 +++++ .../catalog-retrieval/round3/build_report3.py | 510 ++++++++++++++++++ .../round3/test_build_report3.py | 406 ++++++++++++++ 3 files changed, 1044 insertions(+) create mode 100644 evals/catalog-retrieval/round3/VERDICT.md create mode 100644 evals/catalog-retrieval/round3/build_report3.py create mode 100644 evals/catalog-retrieval/round3/test_build_report3.py diff --git a/evals/catalog-retrieval/round3/VERDICT.md b/evals/catalog-retrieval/round3/VERDICT.md new file mode 100644 index 0000000000..b9bf80521d --- /dev/null +++ b/evals/catalog-retrieval/round3/VERDICT.md @@ -0,0 +1,128 @@ +# Round 3 verdict, pending + +**Status: pending. This document carries no round 3 result number, because none exists yet.** + +The paid confirmation stage has not run. Everything below that would need a round 3 measurement +is written as an explicit gap with the file that will fill it, rather than as an estimate, a +placeholder or a plausible-looking number. A verdict written before the data is a description of +an expectation, not a finding. + +Round 2 numbers appear here only as context and are labelled as such. They are not round 3 +results and must not be read as the ranking. + +## What this round decides + +Which catalog retrieval arm ships as the default for the frame workers: `lexical`, `semantic`, or +`hybrid` at some fusion weight, each swept over list length k. + +Round 3's finding, established before any of this ran, is that the arm choice reduces entirely to +**recall**. Round 2 measured that once the correct move is already on the shown list, every arm +converts it into a good pick at the same rate, between 0.774 and 0.872 with no ordering by arm +family. Nothing distinguishes the arms except whether the answer was on the list. Recall is +decided by the retriever before a single token is bought, so the arm ranking costs nothing and +the paid stage is reduced to the two quantities recall cannot see. + +## The pre-registered rule + +Written and committed in `DECISION-RULE.md` before any round 3 recall number existed, and read +out of that file by `sweep.py` rather than transcribed into code, so editing the rule changes what +the sweep decides. + +1. **Rank on offline recall**, every interval clustered by target move rather than by brief. +2. **If the leader clears the runner-up by more than the clustered 95 percent interval of the + paired difference, that arm ships.** Paired on the same briefs, not a difference of two column + means. +3. **If they do not separate, stop buying data.** The tie breaks on operations, not on the larger + number: in favour of the arm that needs no embedding index and therefore cannot serve stale + vectors. +4. **Then run the paid confirmation** on the top two arms only, for the two quantities recall + cannot see: the refusal rate, and the known-wrong-pick rate that drives the flat 0.5 fit + penalty. If the confirmation contradicts the recall ranking, that is reported as a + contradiction and is not used to reopen step 2. + +The exploratory tier (a reranker, query expansion, anything new) is swept and reported but cannot +win this decision. An exploratory arm that wins is recorded as a follow-up initiative with its +measured margin. + +## The three stop reasons + +The round stops at the first of three conditions, and they are three different verdicts. The +report names which one fired; it never reports them as one. + +| stop reason | what it means | how to read the ranking | +| ----------------- | -------------------------------------------------------------- | ----------------------------------------------------------------------- | +| `separated` | the leader cleared the runner-up under step 2 | an arm won on measurement | +| `waves-exhausted` | all five waves of corpus growth completed without a separation | no arm won; the corpus ran out, and step 3 decides on operations | +| `ceiling-reached` | the spend ceiling fired before the arms separated | no arm won; the budget ran out, and the ranking is what the data bought | + +A fourth state, `still-running`, means none of the three has fired and the report is an interim +read rather than a verdict. **That is the current state.** + +**Which one fired: unknown.** No wave has completed and no round 3 sweep output is committed. +`build_report3.py` computes the stop reason from the sweep outcome, the wave count and the ceiling +flag, and prints it into `report3.html` and `summary3.json`. + +## The ceiling this round is bounded by + +**The shelf holds 424 entries, so no corpus size ever buys more than 424 independent things to be +right about.** Every arm draws from those same 424 entries. Growing the corpus adds briefs, and +briefs written for the same target move are correlated, which is why every interval in this round +is clustered by move and why the effective sample sits below the brief count. Round 2's own +corpus makes the size of that gap concrete: 227 briefs covering 158 distinct target moves carried +roughly 199 briefs' worth of independent information, not 227. More briefs per move buys less +than it appears to. + +The consequence is a hard resolution limit. **If the arms do not separate at 424 entries, the +honest recommendation is a larger catalog, not a larger corpus.** More briefs against the same +shelf cannot manufacture a distinction the shelf does not contain. Step 3 of the rule exists so +that this case ends in a decision rather than in more spending, and the tie then goes to the arm +that needs no embedding index. + +The related result, also a result and not a failure: all arms draw from the same 424 entries, so +at sufficient k they must converge. If the sweep shows that, the finding is that the real question +was **what k**, not which arm, and the report says so and names the point where token cost starts +to outrun the recall gain. + +## What is missing, and what will fill it + +Nothing in this section is estimated. Each row names the artifact that produces the number. + +| missing number | which file produces it | +| ----------------------------------------------------------------- | ------------------------------------------------------------------ | +| recall per arm per k over the round 3 corpus | `report3.html` and `summary3.json`, built by `build_report3.py` | +| the clustered ranking and each arm's best k | `report3.html`, ranking table | +| the paired margin between leader and runner-up, with its interval | `report3.html`, paired table | +| whether step 2 separated, tied, or was not evaluable | `report3.html`, rule outcome line, from `sweep.decide` | +| which arm ships | `report3.html`, rule outcome line | +| the knee, where a longer list stops paying for its tokens | `report3.html`, knee table | +| which of the three stop reasons fired | `report3.html`, stop reason line | +| how many waves completed before the round stopped | the wave count passed to `build_report3.py`; no wave has completed | +| the refusal rate per arm | the paid confirmation stage, not yet run | +| the known-wrong-pick rate per arm | the paid confirmation stage, not yet run | +| whether the confirmation contradicts the recall ranking | this file, once both the sweep and the confirmation exist | + +The two paid quantities are not a formality: round 2 measured refusal at 0.040 for hybrid at k=10 +and 0.110 for semantic at k=5. That is a round 2 number, quoted as the reason the stage exists, +not as a round 3 result. + +## How to finish this document + +1. Run the offline sweep and build the report. Free, local, no model call: + `python3 build_report3.py --waves ` (add `--ceiling-reached` only if the + spend ceiling actually fired). +2. Read the stop reason, the ranking and the paired margin out of `report3.html` and write them + into the gaps above, replacing each "unknown" with the measured number and its interval. +3. Run the paid confirmation on the top two arms only, then record the refusal rate and the + known-wrong-pick rate. +4. If the confirmation contradicts the recall ranking, record it as a contradiction. Do not + quietly resolve it in either direction and do not reopen step 2. +5. Change the status line at the top of this file from pending to the verdict, and state which of + the three stop reasons fired. + +## What is fixed and must not change while this is completed + +- `mountable` and `fit` stay separate numbers and are never collapsed into one score. +- Scoring stays mechanical. No model judges a model. +- Naming the whole shelf still scores 0.0. +- Exactly one variable changes between cells in the paid confirmation. +- The gate regressions carried forward from round 1 and round 2 keep passing. diff --git a/evals/catalog-retrieval/round3/build_report3.py b/evals/catalog-retrieval/round3/build_report3.py new file mode 100644 index 0000000000..b23dd956df --- /dev/null +++ b/evals/catalog-retrieval/round3/build_report3.py @@ -0,0 +1,510 @@ +#!/usr/bin/env python3 +"""Render the round three report from the offline sweep. + +Round 3's finding is that the arm choice reduces to recall, which the retriever +decides before a token is spent. So this report is built from data structures the +sweep already produced: no model call, no network, no spend. + +Three things are deliberately not reinvented. The intervals come from stats.py, +the ranking, knee and rule outcome come from sweep.py, and the report shape comes +from round two's build_report2.py, which already learned not to stack one card per +run. This file only aggregates and renders. + +Every table carries a clustered interval, because a recall number without one +cannot be read against the pre-registered rule. And the report names which of the +three stopping conditions fired, because "the arms separated", "the waves ran out" +and "the ceiling was reached" are three different verdicts. +""" + +import argparse +import html +import json +import math +import os +import sys + +HERE = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, HERE) + +import recall # noqa: E402 +import stats # noqa: E402 +import sweep # noqa: E402 + +import harness2 # noqa: E402 recall puts the round two directory on the path + +REPORT = os.path.join(HERE, "report3.html") +SUMMARY = os.path.join(HERE, "summary3.json") + +# The three ways the round can end, taken from the stopping rule in +# DECISION-RULE.md, plus the honest fourth state for a round still in progress. +# They are kept apart because reporting them as one verdict is the failure this +# report exists to prevent: an arm that won is not an arm that merely outlasted +# the budget. +STOP_REASONS = { + "separated": ( + "the leader cleared the runner-up under step 2, so the round stopped on a " + "measured separation" + ), + "waves-exhausted": ( + "every wave the rule allows completed without a separation, so the round " + "stopped because it ran out of corpus, not because an arm won" + ), + "ceiling-reached": ( + "the spend ceiling was reached before the arms separated, so the round " + "stopped on budget and the ranking is whatever the data bought" + ), + "still-running": ( + "none of the three stopping conditions has fired yet, so this is an " + "interim read and not a verdict" + ), +} + + +def stop_reason(outcome, waves_run, rule, ceiling_reached=False): + """Which stopping condition fired, in the order the rule lists them. + + Separation is checked first because the rule evaluates it after each wave, so + a round that separates on its final wave stopped on the separation and not on + the wave count. + """ + if outcome.get("branch") == "separated": + return "separated" + if ceiling_reached: + return "ceiling-reached" + if waves_run >= rule["max_waves"]: + return "waves-exhausted" + return "still-running" + + +def interval(flags, clusters): + """The clustered interval for one set of per brief hit flags. + + Sorting the ids keeps the interval reproducible when the flags arrive from a + dict, and the cluster key is the target move, never the brief. + """ + ids = sorted(flags) + return stats.clustered_mean([flags[i] for i in ids], [clusters[i] for i in ids]) + + +def curve_rows(swept, clusters): + """One row per arm per k, each with its own interval. + + Round 2 printed the curve as a wide grid, which has no room for error bars and + so quietly ships bare means. Long form costs more rows and is the only shape + where every recall number can carry the interval that makes it readable. + """ + rows = [] + for label in sorted(swept): + for k in sorted(swept[label]["flags"]): + rows.append( + { + "arm": label, + "k": k, + "interval": interval(swept[label]["flags"][k], clusters), + } + ) + return rows + + +def margin_rows(rows, clusters): + """Every ranked arm against the leader, paired brief by brief. + + Subtracting two column means would compare different brief sets. The rule is + written against the paired quantity, so the report shows the paired quantity. + """ + if not rows: + return [] + leader = rows[0] + return [ + { + "arm": row["arm"], + "against": leader["arm"], + "difference": sweep.paired(row, leader, clusters), + } + for row in rows[1:] + ] + + +def knee_rows(swept, clusters, tokens, price): + """Where a longer list stops paying, per arm, with the recall it stops at. + + Unpriced when round two's cost rows are absent: guessing the exchange rate + would turn a measured threshold into a taste call. + """ + if not tokens or not price: + return [] + slope, intercept = tokens + rows = [] + for label in sorted(swept): + arm = swept[label] + at, exhausted = sweep.knee(arm["curve"], slope, price) + if at is None: + continue + rows.append( + { + "arm": label, + "k": at, + "exhausted": exhausted, + "tokens_per_run": round(slope * at + intercept), + "interval": interval(arm["flags"][at], clusters), + } + ) + return rows + + +def _ranked(row): + """A ranking row without its per brief flags, which are bulk, not evidence.""" + return {key: value for key, value in row.items() if key != "flags"} + + +def build_summary( + swept, + unavailable, + clusters, + rule, + shelf_size, + waves_run=0, + ceiling_reached=False, + tokens=None, + price=None, +): + """Everything the report renders, computed once.""" + rows = sweep.rank(swept, clusters, "decision") + exploratory = sweep.rank(swept, clusters, "exploratory") + outcome = sweep.decide(rows, rule, clusters, exploratory) + reason = stop_reason(outcome, waves_run, rule, ceiling_reached) + ks = sorted({k for arm in swept.values() for k in arm["curve"]}) + return { + "shelf_size": shelf_size, + "briefs": len(clusters), + "clusters": len(set(clusters.values())), + "ks": ks, + "waves_run": waves_run, + "max_waves": rule["max_waves"], + "ceiling_reached": ceiling_reached, + "stop_reason": reason, + "stop_reason_says": STOP_REASONS[reason], + "ranking": [_ranked(row) for row in rows], + "exploratory": [_ranked(row) for row in exploratory], + "curve": curve_rows(swept, clusters), + "margins": margin_rows(rows, clusters), + "knees": knee_rows(swept, clusters, tokens, price), + "unavailable": dict(unavailable), + "converged": sweep.converged(swept, ks) if ks else False, + "outcome": { + "branch": outcome["branch"], + "ship": outcome["ship"], + "reason": outcome["reason"], + "difference": outcome.get("difference"), + "followups": [ + {key: value for key, value in followup.items()} + for followup in outcome["followups"] + ], + }, + } + + +def jsonable(value): + """Non-finite bounds become null, because JSON has no infinity. + + A single cluster genuinely bounds nothing, and stats.py says so with an + infinite half width. Writing that literally produces a file only Python can + read back, which is not an auditable artifact. The clusters field on the same + interval already carries why the bound is missing. + """ + if isinstance(value, float): + return value if math.isfinite(value) else None + if isinstance(value, dict): + return {key: jsonable(item) for key, item in value.items()} + if isinstance(value, (list, tuple)): + return [jsonable(item) for item in value] + return value + + +def _table(caption, columns, body_rows): + head = "".join("{0}".format(html.escape(c)) for c in columns) + body = "".join( + "" + "".join("{0}".format(html.escape(str(c))) for c in row) + "" + for row in body_rows + ) + return ( + '' + "{1}{2}
{0}
" + ).format(html.escape(caption), head, body) + + +def _paragraph(text): + return '

{0}

'.format(html.escape(text)) + + +def render(summary): + """The report. Every table ships an interval column, without exception.""" + ranking = [ + [ + place, + row["arm"], + row["family"], + row["k"], + row["interval"]["n"], + row["interval"]["mean"], + stats.format_interval(row["interval"]), + row["interval"]["clusters"], + row["interval"]["icc"], + row["interval"]["n_eff"], + ] + for place, row in enumerate(summary["ranking"], start=1) + ] + curve = [ + [ + row["arm"], + row["k"], + row["interval"]["mean"], + stats.format_interval(row["interval"]), + row["interval"]["clusters"], + row["interval"]["n_eff"], + ] + for row in summary["curve"] + ] + margins = [ + [ + row["arm"], + row["against"], + row["difference"]["n"], + row["difference"]["mean"], + stats.format_interval(row["difference"]), + row["difference"]["clusters"], + row["difference"]["n_eff"], + ] + for row in summary["margins"] + ] + knees = [ + [ + row["arm"], + row["k"], + "no knee within the sweep" if row["exhausted"] else "knee", + row["tokens_per_run"], + row["interval"]["mean"], + stats.format_interval(row["interval"]), + row["interval"]["n_eff"], + ] + for row in summary["knees"] + ] + exploratory = [ + [ + row["arm"], + row["k"], + row["interval"]["mean"], + stats.format_interval(row["interval"]), + row["interval"]["clusters"], + row["interval"]["n_eff"], + ] + for row in summary["exploratory"] + ] + + sections = [ + _table( + "Ranking, decision tier, each arm at its own best k", + ["place", "arm", "family", "k", "n", "recall", "recall 95% ci", + "clusters", "icc", "n eff"], + ranking, + ), + _paragraph( + "Every interval is clustered by target move rather than by brief, because " + "briefs written for the same move are correlated and counting them as " + "independent reports a narrower interval than the corpus supports. N eff is " + "the effective sample clustering leaves. An arm that ran on a single move " + "reads unbounded rather than a misleadingly tight number." + ), + _table( + "Paired against the leader, brief by brief", + ["arm", "against", "n", "difference", "difference 95% ci", + "clusters", "n eff"], + margins, + ), + _paragraph( + "The pairing holds brief difficulty fixed, which a difference of two column " + "means does not. A difference whose interval contains zero has not " + "separated, and step 3 of the rule then decides on operations rather than " + "on the larger number." + ), + _table( + "Recall curve, every arm at every k", + ["arm", "k", "recall", "recall 95% ci", "clusters", "n eff"], + curve, + ), + ] + if knees: + sections.append( + _table( + "Knee, where a longer list stops paying for its tokens", + ["arm", "k", "kind", "input tokens per run", "recall", + "recall 95% ci", "n eff"], + knees, + ) + ) + sections.append( + _paragraph( + "The exchange rate is round two's own: tokens per slot fitted to its " + "cost rows, and tokens per unit recall taken from the full shelf cell. " + "A step pays when the recall it adds is worth more than the tokens it " + "adds at that rate." + ) + ) + else: + sections.append( + _paragraph( + "The knee is not priced: round two's cost rows are unavailable, and " + "guessing the exchange rate would turn a measured threshold into a " + "taste call." + ) + ) + if exploratory: + sections.append( + _table( + "Exploratory tier, reported and never shipped by this round", + ["arm", "k", "recall", "recall 95% ci", "clusters", "n eff"], + exploratory, + ) + ) + for label in sorted(summary["unavailable"]): + sections.append( + _paragraph( + "Arm {0} was not swept: {1}. It is absent from the ranking rather than " + "placed last, because a missing arm that ranks last is " + "indistinguishable from an arm that lost.".format( + label, summary["unavailable"][label] + ) + ) + ) + for followup in summary["outcome"]["followups"]: + sections.append( + _paragraph( + "Follow-up: {0} at k={1} beats {2} by {3:.4f}, 95% {4}. {5}".format( + followup["arm"], + followup["k"], + followup["over"], + followup["margin"]["mean"], + stats.format_interval(followup["margin"]), + followup["note"], + ) + ) + ) + if summary["converged"]: + sections.append( + _paragraph( + "The arms land within a hundredth of each other at k={0}: the open " + "question is what k, not which arm.".format(max(summary["ks"])) + ) + ) + + return """ + +Catalog retrieval, round three + +

Catalog retrieval, round three

+

{briefs} briefs over {clusters} target moves, swept offline against a shelf of +{shelf} entries. No model call and no spend produced any number on this page: recall is decided +by the retriever before a token is bought.

+

Stop reason: {reason}. {says}. Waves run {waves} of {max_waves}, spend ceiling +reached: {ceiling}.

+

Rule outcome: branch {branch}, ship {ship}. {why}

+

Ceiling: the shelf holds {shelf} entries, so no corpus size ever buys more than +{shelf} independent things to be right about. If that resolution proves insufficient, the honest +recommendation is a larger catalog, not a larger corpus.

+{sections} + +""".format( + briefs=summary["briefs"], + clusters=summary["clusters"], + shelf=summary["shelf_size"], + reason=summary["stop_reason"], + says=html.escape(summary["stop_reason_says"]), + waves=summary["waves_run"], + max_waves=summary["max_waves"], + ceiling="yes" if summary["ceiling_reached"] else "no", + branch=html.escape(str(summary["outcome"]["branch"])), + ship=html.escape(str(summary["outcome"]["ship"] or "nothing yet")), + why=html.escape(summary["outcome"]["reason"]), + sections="\n".join(sections), + ) + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--corpus", default=recall.CORPUS) + parser.add_argument("--gold", default=recall.GOLD) + parser.add_argument("--vectors", default=recall.VECTORS) + parser.add_argument("--rule", default=sweep.RULE_FILE) + parser.add_argument("--ks", default=",".join(str(k) for k in sweep.KS)) + parser.add_argument( + "--waves", + type=int, + default=0, + help="waves of corpus growth completed so far, which the stop reason reads", + ) + parser.add_argument( + "--ceiling-reached", + action="store_true", + help="the spend ceiling fired, which is a different verdict from running out of waves", + ) + parser.add_argument("--report", default=REPORT) + parser.add_argument("--summary", default=SUMMARY) + args = parser.parse_args(argv) + + harness2.VECTORS = args.vectors + rule = sweep.parse_rule(args.rule) + ks = tuple(int(k) for k in args.ks.split(",")) + + briefs = recall.load_briefs(args.corpus) + gold = recall.load_gold(args.gold) + entries = harness2.load_shelf() + clusters = sweep.cluster_keys(gold) + arms = { + label: spec + for label, spec in sweep.decision_arms().items() + if spec["family"] in rule["decision_tier"] + } + swept, unavailable = sweep.sweep(briefs, gold, entries, arms, ks) + + summary = build_summary( + swept, + unavailable, + clusters, + rule, + shelf_size=len(entries), + waves_run=args.waves, + ceiling_reached=args.ceiling_reached, + tokens=sweep.token_model(), + price=sweep.recall_price(), + ) + with open(args.summary, "w", encoding="utf-8") as fh: + json.dump(jsonable(summary), fh, indent=2, sort_keys=True, allow_nan=False) + with open(args.report, "w", encoding="utf-8") as fh: + fh.write(render(summary)) + + print("stop reason: {0}".format(summary["stop_reason"])) + print("branch: {0}, ship: {1}".format( + summary["outcome"]["branch"], summary["outcome"]["ship"] or "nothing yet" + )) + for row in summary["ranking"]: + print("{0} at k={1} recall {2:.4f} 95% {3} n_eff {4}".format( + row["arm"], row["k"], row["interval"]["mean"], + stats.format_interval(row["interval"]), row["interval"]["n_eff"], + )) + print("wrote {0} and {1}".format(args.report, args.summary)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/evals/catalog-retrieval/round3/test_build_report3.py b/evals/catalog-retrieval/round3/test_build_report3.py new file mode 100644 index 0000000000..904ef3fedb --- /dev/null +++ b/evals/catalog-retrieval/round3/test_build_report3.py @@ -0,0 +1,406 @@ +#!/usr/bin/env python3 +"""Checks for the round three report builder. + +Unittest style, matching the rest of round three, so one command runs the round: + + cd evals/catalog-retrieval/round3 + python3 -m unittest test_build_report3 -v + +The load-bearing checks are the ones that could catch the report flattering the +round: a table that quietly ships a bare mean, a single cluster rendered as a +tight interval, and a stop reason the reader cannot find. Every assertion about +the rendered page parses the HTML rather than grepping for a string, because a +grep passes on a caption that merely mentions an interval. +""" + +import json +import math +import os +import unittest +from html.parser import HTMLParser +from unittest import mock + +import build_report3 +import stats +import sweep + +import harness2 # noqa: E402 recall puts the round two directory on the path + +RULE = sweep.parse_rule() +SHELF = 424 +# Round two's own fitted exchange rate, so the knee table renders in tests at the +# same prices the report uses in production. +TOKENS = (63.0, 220.0) +PRICE = 23725.0 +KS = (5, 10, 20) + + +class Tables(HTMLParser): + """Every table in a rendered report, as caption, headers and body rows.""" + + def __init__(self): + super().__init__() + self.tables = [] + self._cell = None + self._row = None + + def handle_starttag(self, tag, attrs): + if tag == "table": + self.tables.append({"caption": "", "headers": [], "rows": []}) + elif tag in ("th", "td", "caption"): + self._cell = [] + elif tag == "tr": + self._row = [] + + def handle_data(self, data): + if self._cell is not None: + self._cell.append(data) + + def handle_endtag(self, tag): + if self._cell is not None and tag in ("th", "td", "caption"): + text = "".join(self._cell).strip() + self._cell = None + if tag == "caption": + self.tables[-1]["caption"] = text + elif tag == "th": + self.tables[-1]["headers"].append(text) + elif self._row is not None: + self._row.append(text) + elif tag == "tr" and self._row is not None: + if self._row: + self.tables[-1]["rows"].append(self._row) + self._row = None + + +def tables_in(page): + parser = Tables() + parser.feed(page) + return parser.tables + + +def column(table, header): + """One column of a parsed table, by header name. Raises if the column is absent.""" + index = table["headers"].index(header) + return [row[index] for row in table["rows"]] + + +def corpus(sizes): + """A corpus with sizes[i] briefs written for move i. + + Unequal sizes are the normal case once a wave is interrupted part way, so the + fixture takes them per move rather than assuming a rectangle. + """ + entries = {"move-{0}".format(i): "body {0}".format(i) for i in range(len(sizes))} + briefs, gold = {}, {} + for i, count in enumerate(sizes): + for j in range(count): + brief_id = "{0}-{1}".format(i, j) + briefs[brief_id] = "brief for move {0}".format(i) + gold[brief_id] = {"best": ["move-{0}".format(i)]} + return entries, briefs, gold + + +def hits(predicate): + """An arm that shows the gold move exactly when the predicate says so.""" + + def topk(brief_id, brief, entries, k): + return ["move-{0}".format(brief_id.split("-")[0])] if predicate(brief_id) else [] + + return topk + + +def arm(fn, family, tier="decision"): + return {"family": family, "tier": tier, "fn": fn} + + +ALWAYS = hits(lambda brief_id: True) +NEVER = hits(lambda brief_id: False) +# A near tie: the semantic arm leads by one brief, which no clustered interval +# can call a separation. +NEARLY_ALL = hits(lambda brief_id: brief_id != "0-0") +ONE_FEWER = hits(lambda brief_id: brief_id not in ("0-0", "1-0")) + + +def summarise(sizes, arms, ks=KS, priced=True, **kwargs): + entries, briefs, gold = corpus(sizes) + clusters = sweep.cluster_keys(gold) + swept, unavailable = sweep.sweep(briefs, gold, entries, arms, ks) + return build_report3.build_summary( + swept, + unavailable, + clusters, + RULE, + shelf_size=SHELF, + tokens=TOKENS if priced else None, + price=PRICE if priced else None, + **kwargs + ) + + +def tied_summary(**kwargs): + return summarise( + (3,) * 8, + {"semantic": arm(NEARLY_ALL, "semantic"), "lexical": arm(ONE_FEWER, "lexical")}, + **kwargs + ) + + +class IntervalColumnTests(unittest.TestCase): + """Scenario 1: no table ships a bare mean.""" + + def test_every_rendered_table_carries_an_interval_column(self): + summary = tied_summary() + tables = tables_in(build_report3.render(summary)) + self.assertGreaterEqual(len(tables), 4, [t["caption"] for t in tables]) + for table in tables: + with self.subTest(caption=table["caption"]): + intervals = [h for h in table["headers"] if "95% ci" in h] + self.assertEqual(1, len(intervals), table["headers"]) + + def test_the_interval_cells_hold_intervals_and_not_placeholders(self): + # A column named "95% ci" that renders blanks would pass a header check + # while shipping nothing, so the cells are read too. + summary = tied_summary() + for table in tables_in(build_report3.render(summary)): + header = next(h for h in table["headers"] if "95% ci" in h) + for cell in column(table, header): + with self.subTest(caption=table["caption"], cell=cell): + self.assertTrue( + cell == "unbounded" or " to " in cell, cell + ) + + def test_the_ranking_interval_matches_the_estimator_rather_than_being_restated(self): + # The rendered number has to be the one stats.py computed, not a rounding + # of the mean dressed up as an interval. + summary = tied_summary() + table = next( + t for t in tables_in(build_report3.render(summary)) if "Ranking" in t["caption"] + ) + rendered = column(table, "recall 95% ci") + expected = [stats.format_interval(r["interval"]) for r in summary["ranking"]] + self.assertEqual(expected, rendered) + + +class EmDashTests(unittest.TestCase): + """Scenario 2: the house rule applies to the source and to what it renders.""" + + FILES = ("build_report3.py", "test_build_report3.py", "VERDICT.md") + # Written as a code point rather than as the character, so this file can + # assert the rule without breaking it on itself. + EM_DASH = chr(0x2014) + + def test_no_file_this_unit_ships_carries_an_em_dash(self): + for name in self.FILES: + path = os.path.join(build_report3.HERE, name) + with self.subTest(file=name): + self.assertTrue(os.path.exists(path), path) + with open(path, encoding="utf-8") as fh: + self.assertEqual(0, fh.read().count(self.EM_DASH), name) + + def test_the_rendered_report_carries_no_em_dash(self): + page = build_report3.render(tied_summary()) + self.assertEqual(0, page.count(self.EM_DASH), "rendered report") + + +class SingleClusterTests(unittest.TestCase): + """Scenario 3: one cluster bounds nothing, and the report must say so.""" + + def setUp(self): + # Three briefs, all written for the same target move. Treating them as + # three independent observations is the exact over-precision the + # clustering exists to stop. + self.summary = summarise( + (3,), {"semantic": arm(ALWAYS, "semantic"), "lexical": arm(NEVER, "lexical")} + ) + + def test_the_single_cluster_interval_is_unbounded_not_tight(self): + for row in self.summary["ranking"]: + self.assertEqual(1, row["interval"]["clusters"], row) + self.assertFalse(math.isfinite(row["interval"]["hi"]), row["interval"]) + self.assertEqual(1.0, row["interval"]["n_eff"], row["interval"]) + + def test_the_rendered_cell_reads_unbounded(self): + table = next( + t + for t in tables_in(build_report3.render(self.summary)) + if "Ranking" in t["caption"] + ) + self.assertEqual( + ["unbounded"] * len(self.summary["ranking"]), column(table, "recall 95% ci") + ) + + def test_a_single_cluster_never_fires_the_separation_branch(self): + # One arm hits every brief and the other misses every brief, which is the + # largest gap a corpus can show. On one cluster it still cannot separate, + # and a report that shipped an arm here would be shipping noise. + self.assertNotEqual("separated", self.summary["outcome"]["branch"]) + self.assertNotEqual("separated", self.summary["stop_reason"]) + + def test_the_summary_file_stays_strict_json_when_a_bound_is_infinite(self): + # JSON has no infinity. Writing one produces a file only Python can read + # back, which is not an auditable artifact. + text = json.dumps( + build_report3.jsonable(self.summary), sort_keys=True, allow_nan=False + ) + reloaded = json.loads(text) + interval = reloaded["ranking"][0]["interval"] + self.assertIsNone(interval["hi"]) + self.assertEqual(1, interval["clusters"]) + + +class StopReasonTests(unittest.TestCase): + """Scenario 4: three different verdicts, never reported as one.""" + + def separated_summary(self, **kwargs): + return summarise( + (3,) * 8, + {"semantic": arm(ALWAYS, "semantic"), "lexical": arm(NEVER, "lexical")}, + **kwargs + ) + + def assert_reason_is_readable(self, summary, expected): + self.assertEqual(expected, summary["stop_reason"]) + page = build_report3.render(summary) + self.assertIn(expected, page) + self.assertIn(build_report3.STOP_REASONS[expected].split(",")[0], page) + + def test_a_separation_stops_the_round_on_the_separation(self): + summary = self.separated_summary(waves_run=2) + self.assertEqual("separated", summary["outcome"]["branch"]) + self.assert_reason_is_readable(summary, "separated") + + def test_running_out_of_waves_is_not_reported_as_a_win(self): + summary = tied_summary(waves_run=RULE["max_waves"]) + self.assertEqual("tie", summary["outcome"]["branch"]) + self.assert_reason_is_readable(summary, "waves-exhausted") + + def test_the_spend_ceiling_is_its_own_verdict(self): + summary = tied_summary(waves_run=2, ceiling_reached=True) + self.assert_reason_is_readable(summary, "ceiling-reached") + + def test_a_round_that_has_not_stopped_says_so_rather_than_implying_a_verdict(self): + self.assert_reason_is_readable(tied_summary(waves_run=1), "still-running") + + def test_a_separation_on_the_final_wave_reads_as_a_separation(self): + # The rule evaluates step 2 after each wave, so a round that separates on + # its last wave stopped on the separation and not on the wave count. + summary = self.separated_summary(waves_run=RULE["max_waves"], ceiling_reached=True) + self.assert_reason_is_readable(summary, "separated") + + def test_the_shelf_ceiling_is_stated_in_the_report_itself(self): + page = build_report3.render(tied_summary()) + self.assertIn("{0} independent things to be right about".format(SHELF), page) + self.assertIn("larger catalog, not a larger corpus", page) + + +class InterruptedWaveTests(unittest.TestCase): + """Scenario 5: unequal cluster sizes, which an interrupted wave always leaves.""" + + SIZES = (3, 2, 1, 3, 2, 1, 3, 2, 1, 2) + + def setUp(self): + # The leader hits every brief for an even numbered move and misses every + # brief for an odd one, so all the variance sits between clusters. That is + # the case where ragged cluster sizes have to cost the interval + # information, and where an unclustered mean would over-report. + self.summary = summarise( + self.SIZES, + { + "semantic": arm(hits(lambda b: int(b.split("-")[0]) % 2 == 0), "semantic"), + "lexical": arm(hits(lambda b: b.startswith("0-")), "lexical"), + }, + ) + + def test_the_report_renders_over_unequal_clusters(self): + tables = tables_in(build_report3.render(self.summary)) + captions = [t["caption"] for t in tables] + self.assertTrue(any("Ranking" in c for c in captions), captions) + self.assertTrue(any("Recall curve" in c for c in captions), captions) + self.assertTrue(any("Paired" in c for c in captions), captions) + self.assertTrue(any("Knee" in c for c in captions), captions) + for table in tables: + self.assertTrue(table["rows"], table["caption"]) + + def test_the_interval_still_costs_information_when_the_sizes_are_ragged(self): + row = self.summary["ranking"][0] + self.assertEqual(sum(self.SIZES), row["interval"]["n"]) + self.assertEqual(len(self.SIZES), row["interval"]["clusters"]) + self.assertLess(row["interval"]["n_eff"], row["interval"]["n"]) + self.assertTrue(math.isfinite(row["interval"]["hi"])) + + def test_the_curve_carries_one_row_per_arm_per_k(self): + rows = self.summary["curve"] + self.assertEqual(2 * len(KS), len(rows)) + self.assertEqual(set(KS), {row["k"] for row in rows}) + + def test_the_paired_margin_is_measured_on_the_shared_briefs(self): + margin = self.summary["margins"][0] + self.assertEqual(sum(self.SIZES), margin["difference"]["n"]) + self.assertEqual(self.summary["ranking"][0]["arm"], margin["against"]) + + +class DegradedInputTests(unittest.TestCase): + """Error paths the five scenarios do not reach.""" + + def test_no_available_arm_renders_a_not_evaluable_report_rather_than_crashing(self): + summary = summarise((3, 2, 1), {}) + self.assertEqual("not-evaluable", summary["outcome"]["branch"]) + self.assertIsNone(summary["outcome"]["ship"]) + page = build_report3.render(summary) + self.assertIn("not-evaluable", page) + self.assertIn("nothing yet", page) + # Still a report: the tables are present and empty rather than absent. + for table in tables_in(page): + self.assertTrue([h for h in table["headers"] if "95% ci" in h], table) + + def test_a_missing_embedding_index_is_reported_as_missing_not_as_a_loss(self): + entries, briefs, gold = corpus((3, 3, 3)) + clusters = sweep.cluster_keys(gold) + with mock.patch.object(harness2, "VECTORS", "/nonexistent/vectors.json"): + arms = { + "lexical": arm(ALWAYS, "lexical"), + "semantic": sweep.decision_arms()["semantic"], + } + swept, unavailable = sweep.sweep(briefs, gold, entries, arms, KS) + summary = build_report3.build_summary( + swept, unavailable, clusters, RULE, shelf_size=SHELF + ) + self.assertEqual(["semantic"], sorted(summary["unavailable"])) + self.assertEqual(["lexical"], [row["arm"] for row in summary["ranking"]]) + page = build_report3.render(summary) + self.assertIn("Arm semantic was not swept", page) + self.assertIn("indistinguishable from an arm that lost", page) + + def test_an_unpriced_knee_says_so_instead_of_guessing_the_exchange_rate(self): + summary = tied_summary(priced=False) + self.assertEqual([], summary["knees"]) + page = build_report3.render(summary) + self.assertIn("The knee is not priced", page) + self.assertFalse(any("Knee" in t["caption"] for t in tables_in(page))) + + def test_an_exploratory_winner_is_a_follow_up_and_not_the_shipped_arm(self): + summary = summarise( + (3,) * 8, + { + "lexical": arm(ONE_FEWER, "lexical"), + "semantic": arm(NEVER, "semantic"), + "reranker": arm(ALWAYS, "reranker", tier="exploratory"), + }, + ) + self.assertEqual(["reranker"], [f["arm"] for f in summary["outcome"]["followups"]]) + self.assertNotEqual("reranker", summary["outcome"]["ship"]) + page = build_report3.render(summary) + self.assertIn("Follow-up: reranker", page) + self.assertIn("not shipped", page) + + def test_converged_arms_say_the_question_was_what_k(self): + summary = summarise((3,) * 8, { + "lexical": arm(ALWAYS, "lexical"), + "semantic": arm(ALWAYS, "semantic"), + }) + self.assertTrue(summary["converged"]) + self.assertIn("the open question is what k, not which arm", build_report3.render(summary)) + + +if __name__ == "__main__": + unittest.main(verbosity=2)