diff --git a/evals/catalog-retrieval/round2/gen_briefs.py b/evals/catalog-retrieval/round2/gen_briefs.py index ff0794456b..de86d7845c 100644 --- a/evals/catalog-retrieval/round2/gen_briefs.py +++ b/evals/catalog-retrieval/round2/gen_briefs.py @@ -302,7 +302,13 @@ def distractor_gate(brief_text, name, entries): query = set(harness2.tokens(brief_text)) target_score = _lexical_score(query, entries[name]) if target_score == 0.0: - return 0, True + # No band exists to count competitors in, so this is not a count of + # zero. Reported as None to keep it distinguishable: a zero means the + # brief has a band and stands alone in it, which is a brief that gives + # its move away, while this means the brief shares no vocabulary with + # its own entry at all. Thin entries fail this way whatever the brief + # says, and reading both as "distractor:0" hides that. + return None, True floor = NEIGHBOR_SCORE_RATIO * target_score count = sum( 1 @@ -331,6 +337,8 @@ def validate_candidate(name, brief_text, entries): return f"vocabulary:{overlap:.6f}" competitors, isolated = distractor_gate(brief_text, name, entries) if isolated: + if competitors is None: + return "distractor:no_shelf_overlap" return f"distractor:{competitors}" return None diff --git a/evals/catalog-retrieval/round2/test_briefs.py b/evals/catalog-retrieval/round2/test_briefs.py index 07592a6487..1783023aa0 100644 --- a/evals/catalog-retrieval/round2/test_briefs.py +++ b/evals/catalog-retrieval/round2/test_briefs.py @@ -103,6 +103,36 @@ class BriefGeneratorTests(unittest.TestCase): self.assertEqual(0, count) self.assertTrue(isolated) + def test_no_overlap_is_reported_apart_from_a_competitor_count_of_zero(self): + """Both reject, for opposite reasons, so they must not print the same. + + A count of zero means the brief has a score band and is alone in it, + which is a brief that gives its move away. No overlap means the brief + never scored against its own entry, which is what a thin entry does to + every brief written for it. Reading both as "distractor:0" sent a real + diagnosis of the second down the path of the first. + """ + synthetic = { + "target": "orchid saffron", + "other-a": "harbor cobalt", + "other-b": "lantern quartz", + } + count, isolated = briefs.distractor_gate( + "Nothing here shares a single word with that entry.", + "target", + synthetic, + ) + self.assertIsNone(count) + self.assertTrue(isolated) + self.assertEqual( + "distractor:no_shelf_overlap", + briefs.validate_candidate( + "target", + "Nothing here shares a single word with that entry.", + synthetic, + ), + ) + def test_name_forms_are_caught_and_never_survive_an_accepted_corpus(self): name = "caption-camera-follow" for leaked in ( diff --git a/evals/catalog-retrieval/round2/test_validate_gold.py b/evals/catalog-retrieval/round2/test_validate_gold.py index e807c559d3..0a8021f2b2 100644 --- a/evals/catalog-retrieval/round2/test_validate_gold.py +++ b/evals/catalog-retrieval/round2/test_validate_gold.py @@ -436,6 +436,41 @@ class ValidateGoldTests(unittest.TestCase): ], ) + def test_a_checkpoint_from_another_corpus_is_reread_not_inherited(self): + """Same ids, different briefs: the verdicts must not carry across. + + Brief ids are positions in a corpus, so a regenerated corpus numbers + from the start again and every id collides with an older brief that + says something else. Keyed on the id alone, resume answered the new + corpus with the old corpus's verdicts and never spent a call to notice, + and the audit then pruned the new corpus on them. + """ + first_briefs, gold_by_id = corpus(3) + replies = [canned("move-alpha", "judged for the first corpus")] * 9 + with patch.object(validate_gold.provider, "chat", side_effect=replies): + validate_gold.evaluate_corpus(first_briefs, gold_by_id, SHELF) + + with open(self.checkpoint, encoding="utf-8") as fh: + self.assertEqual(len([line for line in fh if line.strip()]), 3) + + # Same ids, text a committee has never seen. + second_briefs = { + brief_id: f"a different corpus reused id {brief_id}" + for brief_id in first_briefs + } + rest = [canned("move-alpha", "judged for the second corpus")] * 9 + with patch.object( + validate_gold.provider, "chat", side_effect=rest + ) as chat_mock: + result = validate_gold.evaluate_corpus(second_briefs, gold_by_id, SHELF) + + self.assertEqual(result["new_reads"], 3) + self.assertEqual(chat_mock.call_count, 9) + self.assertEqual( + {record["passes"][0]["why"] for record in result["records"]}, + {"judged for the second corpus"}, + ) + def test_ceiling_below_projection_stops_before_any_read(self): briefs, gold_by_id = corpus(4) projection = validate_gold.project_committee(briefs, gold_by_id, SHELF) diff --git a/evals/catalog-retrieval/round2/validate_gold.py b/evals/catalog-retrieval/round2/validate_gold.py index ccc99020b0..db7da83bba 100644 --- a/evals/catalog-retrieval/round2/validate_gold.py +++ b/evals/catalog-retrieval/round2/validate_gold.py @@ -3,6 +3,7 @@ import collections import glob +import hashlib import json import math import os @@ -39,7 +40,14 @@ DEFAULT_SAMPLE_RATE = 1.0 # with three readers projects near $32; this leaves headroom without letting a # mis-set rate spend unbounded. DEFAULT_MAX_USD = 40.00 -DEFAULT_COMMITTEE_CHECKPOINT = os.path.join(HERE, "checkpoints", "committee.jsonl") +# Under the corpus, for the same reason the audit is. A checkpoint records what +# a committee said about particular briefs, so it belongs to the corpus those +# briefs came from. At HERE it was shared by every round, and resume reads it by +# brief id, so a later corpus numbering from 001 again would inherit this +# corpus's verdicts for briefs that committee never saw. +DEFAULT_COMMITTEE_CHECKPOINT = os.path.join( + harness2.CORPUS_ROOT, "checkpoints", "committee.jsonl" +) COMMITTEE_INSTRUCTIONS = """Reconstruct which single catalog move the brief describes. @@ -51,6 +59,10 @@ Do not discuss any prior response or any evaluation result. """ +def _brief_digest(text): + return hashlib.sha256(text.encode("utf-8")).hexdigest() + + def build_committee_context(brief, shelf): """Build the complete pass context from one brief and the full shelf only.""" rendered_shelf = "\n\n".join( @@ -315,11 +327,20 @@ def evaluate_corpus( "EVAL_COMMITTEE_CHECKPOINT", DEFAULT_COMMITTEE_CHECKPOINT ) kill_path = kill_path or os.environ.get("EVAL_KILL_FILE", run2.DEFAULT_KILL_FILE) - judged = { - record["brief_id"]: record - for record in run2.load_checkpoint(checkpoint_path) - if isinstance(record.get("brief_id"), str) - } + # Resume on what the committee actually read, not on where it sat in the + # corpus. A brief id is a position: regenerate a corpus and 001 is a + # different brief that reuses the name. Keyed on the id alone, resume would + # hand those verdicts to briefs no committee ever saw, silently, and the + # audit would prune on them. A record with no digest predates this check and + # cannot prove which brief it judged, so it is re-read rather than trusted. + judged = {} + for record in run2.load_checkpoint(checkpoint_path): + brief_id = record.get("brief_id") + if not isinstance(brief_id, str) or brief_id not in briefs: + continue + if record.get("brief_digest") != _brief_digest(briefs[brief_id]): + continue + judged[brief_id] = record total_usd = sum( _record_cost(record, model) for brief_id, record in judged.items() @@ -364,6 +385,7 @@ def evaluate_corpus( record = { "brief_id": brief_id, + "brief_digest": _brief_digest(briefs[brief_id]), "constructed_gold": list(constructed_gold), "effective_gold": effective_gold, "passes": passes,