From 10b1a89aeedb2384697b4eaa4061ea7791671ccd Mon Sep 17 00:00:00 2001 From: Miguel Angel Simon Sierra Date: Sat, 1 Aug 2026 02:32:15 +0200 Subject: [PATCH] fix(evals): key committee resume on the brief, not its position The committee checkpoint defaulted under HERE while the briefs, gold and audit it belongs to had already moved under CORPUS_ROOT, so every corpus shared one file. Resume then matched on brief_id alone. Ids are positions: a regenerated corpus numbers from the start again, so each id collides with an older brief that says something else, and resume answered the new corpus with the old corpus's verdicts without spending a call to notice. The audit prunes on those verdicts, so the corpus would have been cut on judgements of briefs no committee ever read. The checkpoint now sits under CORPUS_ROOT and every record carries the sha256 of the brief text it judged. A record whose digest does not match the brief now on disk is re-read, and one written before this change cannot prove which brief it saw, so it is re-read too. Also splits the brief gate's distractor rejection in two. A competitor count of zero means the brief has a score band and stands alone in it, which is a brief that gives its move away. No overlap with its own entry means it never scored at all, which is what a thin shelf entry does to every brief written for it whatever the brief says. Both printed "distractor:0", which sent a diagnosis of the second down the path of the first. Tests cover both branches, and the cross-corpus case fails against the old resume with new_reads 0 where it should be 3. --- evals/catalog-retrieval/round2/gen_briefs.py | 10 +++++- evals/catalog-retrieval/round2/test_briefs.py | 30 ++++++++++++++++ .../round2/test_validate_gold.py | 35 +++++++++++++++++++ .../catalog-retrieval/round2/validate_gold.py | 34 ++++++++++++++---- 4 files changed, 102 insertions(+), 7 deletions(-) diff --git a/evals/catalog-retrieval/round2/gen_briefs.py b/evals/catalog-retrieval/round2/gen_briefs.py index ff0794456b..de86d7845c 100644 --- a/evals/catalog-retrieval/round2/gen_briefs.py +++ b/evals/catalog-retrieval/round2/gen_briefs.py @@ -302,7 +302,13 @@ def distractor_gate(brief_text, name, entries): query = set(harness2.tokens(brief_text)) target_score = _lexical_score(query, entries[name]) if target_score == 0.0: - return 0, True + # No band exists to count competitors in, so this is not a count of + # zero. Reported as None to keep it distinguishable: a zero means the + # brief has a band and stands alone in it, which is a brief that gives + # its move away, while this means the brief shares no vocabulary with + # its own entry at all. Thin entries fail this way whatever the brief + # says, and reading both as "distractor:0" hides that. + return None, True floor = NEIGHBOR_SCORE_RATIO * target_score count = sum( 1 @@ -331,6 +337,8 @@ def validate_candidate(name, brief_text, entries): return f"vocabulary:{overlap:.6f}" competitors, isolated = distractor_gate(brief_text, name, entries) if isolated: + if competitors is None: + return "distractor:no_shelf_overlap" return f"distractor:{competitors}" return None diff --git a/evals/catalog-retrieval/round2/test_briefs.py b/evals/catalog-retrieval/round2/test_briefs.py index 07592a6487..1783023aa0 100644 --- a/evals/catalog-retrieval/round2/test_briefs.py +++ b/evals/catalog-retrieval/round2/test_briefs.py @@ -103,6 +103,36 @@ class BriefGeneratorTests(unittest.TestCase): self.assertEqual(0, count) self.assertTrue(isolated) + def test_no_overlap_is_reported_apart_from_a_competitor_count_of_zero(self): + """Both reject, for opposite reasons, so they must not print the same. + + A count of zero means the brief has a score band and is alone in it, + which is a brief that gives its move away. No overlap means the brief + never scored against its own entry, which is what a thin entry does to + every brief written for it. Reading both as "distractor:0" sent a real + diagnosis of the second down the path of the first. + """ + synthetic = { + "target": "orchid saffron", + "other-a": "harbor cobalt", + "other-b": "lantern quartz", + } + count, isolated = briefs.distractor_gate( + "Nothing here shares a single word with that entry.", + "target", + synthetic, + ) + self.assertIsNone(count) + self.assertTrue(isolated) + self.assertEqual( + "distractor:no_shelf_overlap", + briefs.validate_candidate( + "target", + "Nothing here shares a single word with that entry.", + synthetic, + ), + ) + def test_name_forms_are_caught_and_never_survive_an_accepted_corpus(self): name = "caption-camera-follow" for leaked in ( diff --git a/evals/catalog-retrieval/round2/test_validate_gold.py b/evals/catalog-retrieval/round2/test_validate_gold.py index e807c559d3..0a8021f2b2 100644 --- a/evals/catalog-retrieval/round2/test_validate_gold.py +++ b/evals/catalog-retrieval/round2/test_validate_gold.py @@ -436,6 +436,41 @@ class ValidateGoldTests(unittest.TestCase): ], ) + def test_a_checkpoint_from_another_corpus_is_reread_not_inherited(self): + """Same ids, different briefs: the verdicts must not carry across. + + Brief ids are positions in a corpus, so a regenerated corpus numbers + from the start again and every id collides with an older brief that + says something else. Keyed on the id alone, resume answered the new + corpus with the old corpus's verdicts and never spent a call to notice, + and the audit then pruned the new corpus on them. + """ + first_briefs, gold_by_id = corpus(3) + replies = [canned("move-alpha", "judged for the first corpus")] * 9 + with patch.object(validate_gold.provider, "chat", side_effect=replies): + validate_gold.evaluate_corpus(first_briefs, gold_by_id, SHELF) + + with open(self.checkpoint, encoding="utf-8") as fh: + self.assertEqual(len([line for line in fh if line.strip()]), 3) + + # Same ids, text a committee has never seen. + second_briefs = { + brief_id: f"a different corpus reused id {brief_id}" + for brief_id in first_briefs + } + rest = [canned("move-alpha", "judged for the second corpus")] * 9 + with patch.object( + validate_gold.provider, "chat", side_effect=rest + ) as chat_mock: + result = validate_gold.evaluate_corpus(second_briefs, gold_by_id, SHELF) + + self.assertEqual(result["new_reads"], 3) + self.assertEqual(chat_mock.call_count, 9) + self.assertEqual( + {record["passes"][0]["why"] for record in result["records"]}, + {"judged for the second corpus"}, + ) + def test_ceiling_below_projection_stops_before_any_read(self): briefs, gold_by_id = corpus(4) projection = validate_gold.project_committee(briefs, gold_by_id, SHELF) diff --git a/evals/catalog-retrieval/round2/validate_gold.py b/evals/catalog-retrieval/round2/validate_gold.py index ccc99020b0..db7da83bba 100644 --- a/evals/catalog-retrieval/round2/validate_gold.py +++ b/evals/catalog-retrieval/round2/validate_gold.py @@ -3,6 +3,7 @@ import collections import glob +import hashlib import json import math import os @@ -39,7 +40,14 @@ DEFAULT_SAMPLE_RATE = 1.0 # with three readers projects near $32; this leaves headroom without letting a # mis-set rate spend unbounded. DEFAULT_MAX_USD = 40.00 -DEFAULT_COMMITTEE_CHECKPOINT = os.path.join(HERE, "checkpoints", "committee.jsonl") +# Under the corpus, for the same reason the audit is. A checkpoint records what +# a committee said about particular briefs, so it belongs to the corpus those +# briefs came from. At HERE it was shared by every round, and resume reads it by +# brief id, so a later corpus numbering from 001 again would inherit this +# corpus's verdicts for briefs that committee never saw. +DEFAULT_COMMITTEE_CHECKPOINT = os.path.join( + harness2.CORPUS_ROOT, "checkpoints", "committee.jsonl" +) COMMITTEE_INSTRUCTIONS = """Reconstruct which single catalog move the brief describes. @@ -51,6 +59,10 @@ Do not discuss any prior response or any evaluation result. """ +def _brief_digest(text): + return hashlib.sha256(text.encode("utf-8")).hexdigest() + + def build_committee_context(brief, shelf): """Build the complete pass context from one brief and the full shelf only.""" rendered_shelf = "\n\n".join( @@ -315,11 +327,20 @@ def evaluate_corpus( "EVAL_COMMITTEE_CHECKPOINT", DEFAULT_COMMITTEE_CHECKPOINT ) kill_path = kill_path or os.environ.get("EVAL_KILL_FILE", run2.DEFAULT_KILL_FILE) - judged = { - record["brief_id"]: record - for record in run2.load_checkpoint(checkpoint_path) - if isinstance(record.get("brief_id"), str) - } + # Resume on what the committee actually read, not on where it sat in the + # corpus. A brief id is a position: regenerate a corpus and 001 is a + # different brief that reuses the name. Keyed on the id alone, resume would + # hand those verdicts to briefs no committee ever saw, silently, and the + # audit would prune on them. A record with no digest predates this check and + # cannot prove which brief it judged, so it is re-read rather than trusted. + judged = {} + for record in run2.load_checkpoint(checkpoint_path): + brief_id = record.get("brief_id") + if not isinstance(brief_id, str) or brief_id not in briefs: + continue + if record.get("brief_digest") != _brief_digest(briefs[brief_id]): + continue + judged[brief_id] = record total_usd = sum( _record_cost(record, model) for brief_id, record in judged.items() @@ -364,6 +385,7 @@ def evaluate_corpus( record = { "brief_id": brief_id, + "brief_digest": _brief_digest(briefs[brief_id]), "constructed_gold": list(constructed_gold), "effective_gold": effective_gold, "passes": passes,