diff --git a/evals/catalog-retrieval/round2/test_validate_gold.py b/evals/catalog-retrieval/round2/test_validate_gold.py index 7902998d22..86c2385a8c 100644 --- a/evals/catalog-retrieval/round2/test_validate_gold.py +++ b/evals/catalog-retrieval/round2/test_validate_gold.py @@ -308,6 +308,42 @@ class ValidateGoldTests(unittest.TestCase): True, ) + def test_projection_prices_the_batch_tier_beside_the_interactive_one(self): + # The committee is the dominant cost of a scaled round and has no + # latency requirement, so quoting only the interactive tier at the gate + # would overstate the round by about two times. + briefs, gold_by_id = corpus(40) + lines = [] + projection = validate_gold.committee_dry_run( + briefs, + gold_by_id, + SHELF, + sample_rate=validate_gold.ROUND2_SAMPLE_RATE, + emit=lines.append, + ) + + self.assertIsNotNone(projection["batch_usd"]) + self.assertLess(projection["batch_usd"], projection["projected_usd"]) + self.assertAlmostEqual( + projection["batch_usd"], projection["projected_usd"] / 2, places=6 + ) + self.assertIn("Same reads on the batch tier: $", "\n".join(lines)) + + def test_projection_omits_the_batch_line_for_a_model_without_that_tier(self): + briefs, gold_by_id = corpus(4) + lines = [] + with patch.dict( + validate_gold.run2.PRICE_TABLE, + {"solo-model": {"input": 1.0, "cached": 0.1, "output": 5.0}}, + clear=False, + ): + projection = validate_gold.committee_dry_run( + briefs, gold_by_id, SHELF, model="solo-model", emit=lines.append + ) + + self.assertIsNone(projection["batch_usd"]) + self.assertNotIn("batch tier", "\n".join(lines)) + def test_projected_reads_are_rate_times_corpus_times_readers(self): for size, rate in ((40, 0.20), (10, 0.30), (1360, 0.20), (7, 1.0)): with self.subTest(size=size, rate=rate): diff --git a/evals/catalog-retrieval/round2/validate_gold.py b/evals/catalog-retrieval/round2/validate_gold.py index bc24b1ac7d..7dc4a95543 100644 --- a/evals/catalog-retrieval/round2/validate_gold.py +++ b/evals/catalog-retrieval/round2/validate_gold.py @@ -185,9 +185,32 @@ def project_committee(briefs, gold_by_id, shelf, sample_rate=DEFAULT_SAMPLE_RATE "output_usd": costs["output"], "projected_usd": projected, "usd_per_read": projected / reads if reads else 0.0, + "batch_usd": _batch_projection(estimated_input, assumed_output, model), } +def _batch_projection(estimated_input, assumed_output, model): + """Same reads on the batch tier, or None when the model has no batch entry. + + The committee is the dominant cost of a scaled round and it has no latency + requirement, so the tier is a real decision rather than a detail. Pricing + only the interactive tier at the gate would overstate the round by roughly + two times and could talk the corpus down for no reason. + """ + batch_model = f"{model}:batch" + if batch_model not in run2.PRICE_TABLE: + return None + costs = run2.usage_cost_breakdown( + { + "prompt_tokens": estimated_input, + "cached_tokens": 0, + "completion_tokens": assumed_output, + }, + batch_model, + ) + return sum(costs.values()) + + def committee_dry_run( briefs, gold_by_id, @@ -229,6 +252,15 @@ def committee_dry_run( f"= ${projection['input_usd']:.6f} input + " f"${projection['output_usd']:.6f} output" ) + if projection["batch_usd"] is not None: + batch_per_read = ( + projection["batch_usd"] / projection["reads"] if projection["reads"] else 0.0 + ) + emit( + f"Same reads on the batch tier: ${projection['batch_usd']:.6f} " + f"(${batch_per_read:.6f} per read). The committee has no latency " + "requirement, so this is the tier a real run should use." + ) emit(f"Ceiling: ${max_usd:.2f}") if projection["projected_usd"] > max_usd: emit(