mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-16 06:19:21 +00:00
feat(eval): quote the batch tier beside the interactive one at the gate
The committee is the dominant cost of a scaled round and it has no latency requirement, so the tier is a real decision rather than a detail. Pricing only the interactive tier overstates the round by about two times, which could talk the corpus size down for no reason.
This commit is contained in:
@@ -308,6 +308,42 @@ class ValidateGoldTests(unittest.TestCase):
|
||||
True,
|
||||
)
|
||||
|
||||
def test_projection_prices_the_batch_tier_beside_the_interactive_one(self):
|
||||
# The committee is the dominant cost of a scaled round and has no
|
||||
# latency requirement, so quoting only the interactive tier at the gate
|
||||
# would overstate the round by about two times.
|
||||
briefs, gold_by_id = corpus(40)
|
||||
lines = []
|
||||
projection = validate_gold.committee_dry_run(
|
||||
briefs,
|
||||
gold_by_id,
|
||||
SHELF,
|
||||
sample_rate=validate_gold.ROUND2_SAMPLE_RATE,
|
||||
emit=lines.append,
|
||||
)
|
||||
|
||||
self.assertIsNotNone(projection["batch_usd"])
|
||||
self.assertLess(projection["batch_usd"], projection["projected_usd"])
|
||||
self.assertAlmostEqual(
|
||||
projection["batch_usd"], projection["projected_usd"] / 2, places=6
|
||||
)
|
||||
self.assertIn("Same reads on the batch tier: $", "\n".join(lines))
|
||||
|
||||
def test_projection_omits_the_batch_line_for_a_model_without_that_tier(self):
|
||||
briefs, gold_by_id = corpus(4)
|
||||
lines = []
|
||||
with patch.dict(
|
||||
validate_gold.run2.PRICE_TABLE,
|
||||
{"solo-model": {"input": 1.0, "cached": 0.1, "output": 5.0}},
|
||||
clear=False,
|
||||
):
|
||||
projection = validate_gold.committee_dry_run(
|
||||
briefs, gold_by_id, SHELF, model="solo-model", emit=lines.append
|
||||
)
|
||||
|
||||
self.assertIsNone(projection["batch_usd"])
|
||||
self.assertNotIn("batch tier", "\n".join(lines))
|
||||
|
||||
def test_projected_reads_are_rate_times_corpus_times_readers(self):
|
||||
for size, rate in ((40, 0.20), (10, 0.30), (1360, 0.20), (7, 1.0)):
|
||||
with self.subTest(size=size, rate=rate):
|
||||
|
||||
@@ -185,9 +185,32 @@ def project_committee(briefs, gold_by_id, shelf, sample_rate=DEFAULT_SAMPLE_RATE
|
||||
"output_usd": costs["output"],
|
||||
"projected_usd": projected,
|
||||
"usd_per_read": projected / reads if reads else 0.0,
|
||||
"batch_usd": _batch_projection(estimated_input, assumed_output, model),
|
||||
}
|
||||
|
||||
|
||||
def _batch_projection(estimated_input, assumed_output, model):
|
||||
"""Same reads on the batch tier, or None when the model has no batch entry.
|
||||
|
||||
The committee is the dominant cost of a scaled round and it has no latency
|
||||
requirement, so the tier is a real decision rather than a detail. Pricing
|
||||
only the interactive tier at the gate would overstate the round by roughly
|
||||
two times and could talk the corpus down for no reason.
|
||||
"""
|
||||
batch_model = f"{model}:batch"
|
||||
if batch_model not in run2.PRICE_TABLE:
|
||||
return None
|
||||
costs = run2.usage_cost_breakdown(
|
||||
{
|
||||
"prompt_tokens": estimated_input,
|
||||
"cached_tokens": 0,
|
||||
"completion_tokens": assumed_output,
|
||||
},
|
||||
batch_model,
|
||||
)
|
||||
return sum(costs.values())
|
||||
|
||||
|
||||
def committee_dry_run(
|
||||
briefs,
|
||||
gold_by_id,
|
||||
@@ -229,6 +252,15 @@ def committee_dry_run(
|
||||
f"= ${projection['input_usd']:.6f} input + "
|
||||
f"${projection['output_usd']:.6f} output"
|
||||
)
|
||||
if projection["batch_usd"] is not None:
|
||||
batch_per_read = (
|
||||
projection["batch_usd"] / projection["reads"] if projection["reads"] else 0.0
|
||||
)
|
||||
emit(
|
||||
f"Same reads on the batch tier: ${projection['batch_usd']:.6f} "
|
||||
f"(${batch_per_read:.6f} per read). The committee has no latency "
|
||||
"requirement, so this is the tier a real run should use."
|
||||
)
|
||||
emit(f"Ceiling: ${max_usd:.2f}")
|
||||
if projection["projected_usd"] > max_usd:
|
||||
emit(
|
||||
|
||||
Reference in New Issue
Block a user