feat(eval): quote the batch tier beside the interactive one at the gate

The committee is the dominant cost of a scaled round and it has no latency
requirement, so the tier is a real decision rather than a detail. Pricing
only the interactive tier overstates the round by about two times, which
could talk the corpus size down for no reason.
This commit is contained in:
Miguel Angel Simon Sierra
2026-07-31 21:16:03 +02:00
parent 3596830187
commit 59abcdb41c
2 changed files with 68 additions and 0 deletions
@@ -308,6 +308,42 @@ class ValidateGoldTests(unittest.TestCase):
True,
)
def test_projection_prices_the_batch_tier_beside_the_interactive_one(self):
# The committee is the dominant cost of a scaled round and has no
# latency requirement, so quoting only the interactive tier at the gate
# would overstate the round by about two times.
briefs, gold_by_id = corpus(40)
lines = []
projection = validate_gold.committee_dry_run(
briefs,
gold_by_id,
SHELF,
sample_rate=validate_gold.ROUND2_SAMPLE_RATE,
emit=lines.append,
)
self.assertIsNotNone(projection["batch_usd"])
self.assertLess(projection["batch_usd"], projection["projected_usd"])
self.assertAlmostEqual(
projection["batch_usd"], projection["projected_usd"] / 2, places=6
)
self.assertIn("Same reads on the batch tier: $", "\n".join(lines))
def test_projection_omits_the_batch_line_for_a_model_without_that_tier(self):
briefs, gold_by_id = corpus(4)
lines = []
with patch.dict(
validate_gold.run2.PRICE_TABLE,
{"solo-model": {"input": 1.0, "cached": 0.1, "output": 5.0}},
clear=False,
):
projection = validate_gold.committee_dry_run(
briefs, gold_by_id, SHELF, model="solo-model", emit=lines.append
)
self.assertIsNone(projection["batch_usd"])
self.assertNotIn("batch tier", "\n".join(lines))
def test_projected_reads_are_rate_times_corpus_times_readers(self):
for size, rate in ((40, 0.20), (10, 0.30), (1360, 0.20), (7, 1.0)):
with self.subTest(size=size, rate=rate):
@@ -185,9 +185,32 @@ def project_committee(briefs, gold_by_id, shelf, sample_rate=DEFAULT_SAMPLE_RATE
"output_usd": costs["output"],
"projected_usd": projected,
"usd_per_read": projected / reads if reads else 0.0,
"batch_usd": _batch_projection(estimated_input, assumed_output, model),
}
def _batch_projection(estimated_input, assumed_output, model):
"""Same reads on the batch tier, or None when the model has no batch entry.
The committee is the dominant cost of a scaled round and it has no latency
requirement, so the tier is a real decision rather than a detail. Pricing
only the interactive tier at the gate would overstate the round by roughly
two times and could talk the corpus down for no reason.
"""
batch_model = f"{model}:batch"
if batch_model not in run2.PRICE_TABLE:
return None
costs = run2.usage_cost_breakdown(
{
"prompt_tokens": estimated_input,
"cached_tokens": 0,
"completion_tokens": assumed_output,
},
batch_model,
)
return sum(costs.values())
def committee_dry_run(
briefs,
gold_by_id,
@@ -229,6 +252,15 @@ def committee_dry_run(
f"= ${projection['input_usd']:.6f} input + "
f"${projection['output_usd']:.6f} output"
)
if projection["batch_usd"] is not None:
batch_per_read = (
projection["batch_usd"] / projection["reads"] if projection["reads"] else 0.0
)
emit(
f"Same reads on the batch tier: ${projection['batch_usd']:.6f} "
f"(${batch_per_read:.6f} per read). The committee has no latency "
"requirement, so this is the tier a real run should use."
)
emit(f"Ceiling: ${max_usd:.2f}")
if projection["projected_usd"] > max_usd:
emit(