diff --git a/evals/catalog-retrieval/round2/gen_briefs.py b/evals/catalog-retrieval/round2/gen_briefs.py index 83220c3a60..a1f0cc1c9d 100644 --- a/evals/catalog-retrieval/round2/gen_briefs.py +++ b/evals/catalog-retrieval/round2/gen_briefs.py @@ -12,6 +12,7 @@ from pathlib import Path import harness2 import provider +import run2 HERE = Path(__file__).resolve().parent @@ -627,10 +628,97 @@ def _fixture_demo(): return 0 +def project_generation(catalog, wave, spent=None, model=None, max_attempts=GEN_MAX_ATTEMPTS): + """Price a generation wave from prompt lengths alone, before any call is made. + + The grid and the committee are both priced before they run. Generation was + not, which left the one cost that scales with the corpus as the only one + nobody could see at the gate. + + Attempts are reported as a floor and a ceiling rather than as a point. A move + costs one call if its first brief passes every gate and up to max_attempts if + it keeps failing them, and the acceptance rate of a wave that has not run is + not something this function can know. Inventing a rate here would turn a + bounded projection into a guess wearing a decimal point. + """ + model = model or MODEL + targets = choose_targets(catalog, wave, spent) + # The same source text generate_corpus loads, not source_of's stratum label: + # the prompt carries the whole artifact, so pricing the label would report a + # cost around three orders of magnitude below the real one. Reading the files + # here also preflights them, so a missing source fails at the gate rather than + # partway through a paid wave. + source_texts = { + name: load_source_text(name, catalog[name]) for name, _ in dict.fromkeys(targets) + } + chars = sum(len(PROMPT.format(source=source_texts[name])) for name, _ in targets) + projection = { + "model": model, + "wave": wave, + "targets": len(targets), + "max_attempts": max_attempts, + "chars_per_attempt_pass": chars, + } + for label, attempts in (("floor", 1), ("ceiling", max_attempts)): + calls = len(targets) * attempts + estimated_input = chars * attempts / run2.CHARS_PER_TOKEN + assumed_output = calls * run2.ASSUMED_OUTPUT_TOKENS + costs = run2.usage_cost_breakdown( + { + "prompt_tokens": estimated_input, + "cached_tokens": 0, + "completion_tokens": assumed_output, + }, + model, + ) + projection[label] = { + "attempts_per_move": attempts, + "calls": calls, + "estimated_input_tokens": estimated_input, + "assumed_output_tokens": assumed_output, + "projected_usd": sum(costs.values()), + } + return projection + + +def generation_dry_run(catalog, wave, spent=None, model=None, emit=print): + """Emit the projection and make no call. Zero network, no key required.""" + projection = project_generation(catalog, wave, spent, model) + emit("DRY RUN: zero network calls and no key required.") + emit( + f"Estimated input tokens use characters divided by {run2.CHARS_PER_TOKEN}; " + "this heuristic is NOT a real tokenizer count." + ) + emit( + f"Assumed output tokens are {run2.ASSUMED_OUTPUT_TOKENS} per call, not " + "measured. Cached tokens are projected as 0 because no requests are made." + ) + emit(f"Model: {projection['model']}") + emit(f"Wave {projection['wave']}: {projection['targets']} targets") + for label in ("floor", "ceiling"): + row = projection[label] + emit( + f"{label.capitalize()} at {row['attempts_per_move']} attempt(s) per move: " + f"{row['calls']} calls, ${row['projected_usd']:.6f}" + ) + emit( + "The true cost sits between them and depends on the acceptance rate, which " + "attempts.jsonl records once a wave has actually run." + ) + return projection + + def main(): if os.environ.get("GEN_BRIEFS_FIXTURE_DEMO") == "1": return _fixture_demo() + if os.environ.get("EVAL_DRY_RUN") == "1": + wave = int(os.environ.get("BRIEF_WAVE", "1")) + catalog = json.loads(CATALOG.read_text(encoding="utf-8")) + spent, _ = load_attempts(ATTEMPTS_LOG) + generation_dry_run(catalog, wave, spent) + return 0 + wave = int(os.environ.get("BRIEF_WAVE", "1")) catalog = json.loads(CATALOG.read_text(encoding="utf-8")) entries = harness2.load_shelf() diff --git a/evals/catalog-retrieval/round2/test_briefs.py b/evals/catalog-retrieval/round2/test_briefs.py index 015b22bd94..07592a6487 100644 --- a/evals/catalog-retrieval/round2/test_briefs.py +++ b/evals/catalog-retrieval/round2/test_briefs.py @@ -481,6 +481,63 @@ class BriefGeneratorTests(unittest.TestCase): calls[0][1], ) + def _priced_catalog(self, root): + """A two move catalog whose sources are real files on disk.""" + catalog = {} + for name, body in (("move-alpha", "alpha " * 400), ("move-beta", "beta")): + path = root / f"{name}.html" + path.write_text(f"", encoding="utf-8") + catalog[name] = { + "sources": "primitive", + "install_path": str(path.relative_to(briefs.REPO_ROOT)), + } + return catalog + + def test_the_projection_prices_the_source_text_not_the_stratum_label(self): + # source_of returns "primitive", the source text is the artifact itself. + # Pricing the label reports a cost far below the real one, and the two are + # only distinguishable when one source is much longer than the other. + with tempfile.TemporaryDirectory(dir=briefs.REPO_ROOT) as temporary: + catalog = self._priced_catalog(Path(temporary)) + projection = briefs.project_generation(catalog, 1) + label_only = 2 * len(briefs.PROMPT.format(source="primitive")) + self.assertEqual(2, projection["targets"]) + self.assertGreater(projection["chars_per_attempt_pass"], label_only * 2) + + def test_the_projection_is_linear_in_waves_and_bounded_by_max_attempts(self): + with tempfile.TemporaryDirectory(dir=briefs.REPO_ROOT) as temporary: + catalog = self._priced_catalog(Path(temporary)) + one = briefs.project_generation(catalog, 1) + two = briefs.project_generation(catalog, 2) + self.assertAlmostEqual( + 2 * one["floor"]["projected_usd"], two["floor"]["projected_usd"], places=9 + ) + self.assertAlmostEqual( + briefs.GEN_MAX_ATTEMPTS * one["floor"]["projected_usd"], + one["ceiling"]["projected_usd"], + places=9, + ) + + def test_the_projection_preflights_a_missing_source_rather_than_pricing_it(self): + catalog = { + "missing-move": { + "sources": "primitive", + "install_path": "evals/catalog-retrieval/round2/no-such-source.html", + } + } + with self.assertRaises(FileNotFoundError): + briefs.project_generation(catalog, 1) + + def test_the_dry_run_emits_both_bounds_and_makes_no_call(self): + lines = [] + with tempfile.TemporaryDirectory(dir=briefs.REPO_ROOT) as temporary: + catalog = self._priced_catalog(Path(temporary)) + briefs.generation_dry_run(catalog, 1, emit=lines.append) + page = "\n".join(lines) + self.assertIn("DRY RUN: zero network calls", page) + self.assertIn("Floor at 1 attempt(s) per move", page) + self.assertIn(f"Ceiling at {briefs.GEN_MAX_ATTEMPTS} attempt(s) per move", page) + def test_missing_source_preflight_makes_no_call_and_writes_nothing(self): catalog = { "missing-move": { diff --git a/evals/catalog-retrieval/round2/test_harness2.py b/evals/catalog-retrieval/round2/test_harness2.py index 098ce85c10..6f7a19f044 100644 --- a/evals/catalog-retrieval/round2/test_harness2.py +++ b/evals/catalog-retrieval/round2/test_harness2.py @@ -137,6 +137,12 @@ class Harness2Tests(unittest.TestCase): entries = round1.load_shelf() briefs = round1.load_briefs() + # Round one's vectors.json is gitignored and lives only where round + # one actually ran, so a fresh checkout has the lexical half of this + # parity check and not the semantic half. Erroring out would report a + # missing untracked input as a broken retriever, and skipping the + # whole test would throw away the half that needs no vectors. + semantic = os.path.exists(round1.VECTORS) with mock.patch.object(harness2, "VECTORS", round1.VECTORS): for brief_id, brief in briefs.items(): self.assertEqual(round1.tokens(brief), harness2.tokens(brief)) @@ -145,10 +151,13 @@ class Harness2Tests(unittest.TestCase): round1.lexical_topk(brief, entries, k), harness2.lexical_topk(brief, entries, k), ) - self.assertEqual( - round1.semantic_topk(brief_id, entries, k), - harness2.semantic_topk(brief_id, entries, k), - ) + if semantic: + self.assertEqual( + round1.semantic_topk(brief_id, entries, k), + harness2.semantic_topk(brief_id, entries, k), + ) + if not semantic: + self.skipTest(f"lexical parity checked; {round1.VECTORS} absent") def test_retriever_k_counts_and_prefix_stability(self): lexical = {