mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-19 11:26:02 +00:00
feat(eval): price a generation wave before it runs
The grid and the gold committee both project their cost before any call is made. Generation did not, which left the one cost that scales linearly with the corpus as the only one invisible at the gate. A wave of 424 briefs was about to be approved on a number nobody had. Attempts are reported as a floor and a ceiling rather than a point. A move costs one call if its first brief clears every gate and up to four if it keeps failing them, and the acceptance rate of a wave that has not run is not knowable here. The projection loads the same source text generate_corpus loads rather than source_of's stratum label. The label is nine characters and the artifact is hundreds, so pricing the label would have understated the input side and the error would have looked like a plausible number rather than a bug. Reading the files also preflights them, so a missing source fails at the gate. Separately, the round one parity check now runs its lexical half and skips only the semantic half when round one's gitignored vectors are absent. It used to error, which reported a missing untracked input as a broken retriever.
This commit is contained in:
@@ -12,6 +12,7 @@ from pathlib import Path
|
||||
|
||||
import harness2
|
||||
import provider
|
||||
import run2
|
||||
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
@@ -627,10 +628,97 @@ def _fixture_demo():
|
||||
return 0
|
||||
|
||||
|
||||
def project_generation(catalog, wave, spent=None, model=None, max_attempts=GEN_MAX_ATTEMPTS):
|
||||
"""Price a generation wave from prompt lengths alone, before any call is made.
|
||||
|
||||
The grid and the committee are both priced before they run. Generation was
|
||||
not, which left the one cost that scales with the corpus as the only one
|
||||
nobody could see at the gate.
|
||||
|
||||
Attempts are reported as a floor and a ceiling rather than as a point. A move
|
||||
costs one call if its first brief passes every gate and up to max_attempts if
|
||||
it keeps failing them, and the acceptance rate of a wave that has not run is
|
||||
not something this function can know. Inventing a rate here would turn a
|
||||
bounded projection into a guess wearing a decimal point.
|
||||
"""
|
||||
model = model or MODEL
|
||||
targets = choose_targets(catalog, wave, spent)
|
||||
# The same source text generate_corpus loads, not source_of's stratum label:
|
||||
# the prompt carries the whole artifact, so pricing the label would report a
|
||||
# cost around three orders of magnitude below the real one. Reading the files
|
||||
# here also preflights them, so a missing source fails at the gate rather than
|
||||
# partway through a paid wave.
|
||||
source_texts = {
|
||||
name: load_source_text(name, catalog[name]) for name, _ in dict.fromkeys(targets)
|
||||
}
|
||||
chars = sum(len(PROMPT.format(source=source_texts[name])) for name, _ in targets)
|
||||
projection = {
|
||||
"model": model,
|
||||
"wave": wave,
|
||||
"targets": len(targets),
|
||||
"max_attempts": max_attempts,
|
||||
"chars_per_attempt_pass": chars,
|
||||
}
|
||||
for label, attempts in (("floor", 1), ("ceiling", max_attempts)):
|
||||
calls = len(targets) * attempts
|
||||
estimated_input = chars * attempts / run2.CHARS_PER_TOKEN
|
||||
assumed_output = calls * run2.ASSUMED_OUTPUT_TOKENS
|
||||
costs = run2.usage_cost_breakdown(
|
||||
{
|
||||
"prompt_tokens": estimated_input,
|
||||
"cached_tokens": 0,
|
||||
"completion_tokens": assumed_output,
|
||||
},
|
||||
model,
|
||||
)
|
||||
projection[label] = {
|
||||
"attempts_per_move": attempts,
|
||||
"calls": calls,
|
||||
"estimated_input_tokens": estimated_input,
|
||||
"assumed_output_tokens": assumed_output,
|
||||
"projected_usd": sum(costs.values()),
|
||||
}
|
||||
return projection
|
||||
|
||||
|
||||
def generation_dry_run(catalog, wave, spent=None, model=None, emit=print):
|
||||
"""Emit the projection and make no call. Zero network, no key required."""
|
||||
projection = project_generation(catalog, wave, spent, model)
|
||||
emit("DRY RUN: zero network calls and no key required.")
|
||||
emit(
|
||||
f"Estimated input tokens use characters divided by {run2.CHARS_PER_TOKEN}; "
|
||||
"this heuristic is NOT a real tokenizer count."
|
||||
)
|
||||
emit(
|
||||
f"Assumed output tokens are {run2.ASSUMED_OUTPUT_TOKENS} per call, not "
|
||||
"measured. Cached tokens are projected as 0 because no requests are made."
|
||||
)
|
||||
emit(f"Model: {projection['model']}")
|
||||
emit(f"Wave {projection['wave']}: {projection['targets']} targets")
|
||||
for label in ("floor", "ceiling"):
|
||||
row = projection[label]
|
||||
emit(
|
||||
f"{label.capitalize()} at {row['attempts_per_move']} attempt(s) per move: "
|
||||
f"{row['calls']} calls, ${row['projected_usd']:.6f}"
|
||||
)
|
||||
emit(
|
||||
"The true cost sits between them and depends on the acceptance rate, which "
|
||||
"attempts.jsonl records once a wave has actually run."
|
||||
)
|
||||
return projection
|
||||
|
||||
|
||||
def main():
|
||||
if os.environ.get("GEN_BRIEFS_FIXTURE_DEMO") == "1":
|
||||
return _fixture_demo()
|
||||
|
||||
if os.environ.get("EVAL_DRY_RUN") == "1":
|
||||
wave = int(os.environ.get("BRIEF_WAVE", "1"))
|
||||
catalog = json.loads(CATALOG.read_text(encoding="utf-8"))
|
||||
spent, _ = load_attempts(ATTEMPTS_LOG)
|
||||
generation_dry_run(catalog, wave, spent)
|
||||
return 0
|
||||
|
||||
wave = int(os.environ.get("BRIEF_WAVE", "1"))
|
||||
catalog = json.loads(CATALOG.read_text(encoding="utf-8"))
|
||||
entries = harness2.load_shelf()
|
||||
|
||||
@@ -481,6 +481,63 @@ class BriefGeneratorTests(unittest.TestCase):
|
||||
calls[0][1],
|
||||
)
|
||||
|
||||
def _priced_catalog(self, root):
|
||||
"""A two move catalog whose sources are real files on disk."""
|
||||
catalog = {}
|
||||
for name, body in (("move-alpha", "alpha " * 400), ("move-beta", "beta")):
|
||||
path = root / f"{name}.html"
|
||||
path.write_text(f"<!--\n{body}\n-->", encoding="utf-8")
|
||||
catalog[name] = {
|
||||
"sources": "primitive",
|
||||
"install_path": str(path.relative_to(briefs.REPO_ROOT)),
|
||||
}
|
||||
return catalog
|
||||
|
||||
def test_the_projection_prices_the_source_text_not_the_stratum_label(self):
|
||||
# source_of returns "primitive", the source text is the artifact itself.
|
||||
# Pricing the label reports a cost far below the real one, and the two are
|
||||
# only distinguishable when one source is much longer than the other.
|
||||
with tempfile.TemporaryDirectory(dir=briefs.REPO_ROOT) as temporary:
|
||||
catalog = self._priced_catalog(Path(temporary))
|
||||
projection = briefs.project_generation(catalog, 1)
|
||||
label_only = 2 * len(briefs.PROMPT.format(source="primitive"))
|
||||
self.assertEqual(2, projection["targets"])
|
||||
self.assertGreater(projection["chars_per_attempt_pass"], label_only * 2)
|
||||
|
||||
def test_the_projection_is_linear_in_waves_and_bounded_by_max_attempts(self):
|
||||
with tempfile.TemporaryDirectory(dir=briefs.REPO_ROOT) as temporary:
|
||||
catalog = self._priced_catalog(Path(temporary))
|
||||
one = briefs.project_generation(catalog, 1)
|
||||
two = briefs.project_generation(catalog, 2)
|
||||
self.assertAlmostEqual(
|
||||
2 * one["floor"]["projected_usd"], two["floor"]["projected_usd"], places=9
|
||||
)
|
||||
self.assertAlmostEqual(
|
||||
briefs.GEN_MAX_ATTEMPTS * one["floor"]["projected_usd"],
|
||||
one["ceiling"]["projected_usd"],
|
||||
places=9,
|
||||
)
|
||||
|
||||
def test_the_projection_preflights_a_missing_source_rather_than_pricing_it(self):
|
||||
catalog = {
|
||||
"missing-move": {
|
||||
"sources": "primitive",
|
||||
"install_path": "evals/catalog-retrieval/round2/no-such-source.html",
|
||||
}
|
||||
}
|
||||
with self.assertRaises(FileNotFoundError):
|
||||
briefs.project_generation(catalog, 1)
|
||||
|
||||
def test_the_dry_run_emits_both_bounds_and_makes_no_call(self):
|
||||
lines = []
|
||||
with tempfile.TemporaryDirectory(dir=briefs.REPO_ROOT) as temporary:
|
||||
catalog = self._priced_catalog(Path(temporary))
|
||||
briefs.generation_dry_run(catalog, 1, emit=lines.append)
|
||||
page = "\n".join(lines)
|
||||
self.assertIn("DRY RUN: zero network calls", page)
|
||||
self.assertIn("Floor at 1 attempt(s) per move", page)
|
||||
self.assertIn(f"Ceiling at {briefs.GEN_MAX_ATTEMPTS} attempt(s) per move", page)
|
||||
|
||||
def test_missing_source_preflight_makes_no_call_and_writes_nothing(self):
|
||||
catalog = {
|
||||
"missing-move": {
|
||||
|
||||
@@ -137,6 +137,12 @@ class Harness2Tests(unittest.TestCase):
|
||||
entries = round1.load_shelf()
|
||||
briefs = round1.load_briefs()
|
||||
|
||||
# Round one's vectors.json is gitignored and lives only where round
|
||||
# one actually ran, so a fresh checkout has the lexical half of this
|
||||
# parity check and not the semantic half. Erroring out would report a
|
||||
# missing untracked input as a broken retriever, and skipping the
|
||||
# whole test would throw away the half that needs no vectors.
|
||||
semantic = os.path.exists(round1.VECTORS)
|
||||
with mock.patch.object(harness2, "VECTORS", round1.VECTORS):
|
||||
for brief_id, brief in briefs.items():
|
||||
self.assertEqual(round1.tokens(brief), harness2.tokens(brief))
|
||||
@@ -145,10 +151,13 @@ class Harness2Tests(unittest.TestCase):
|
||||
round1.lexical_topk(brief, entries, k),
|
||||
harness2.lexical_topk(brief, entries, k),
|
||||
)
|
||||
self.assertEqual(
|
||||
round1.semantic_topk(brief_id, entries, k),
|
||||
harness2.semantic_topk(brief_id, entries, k),
|
||||
)
|
||||
if semantic:
|
||||
self.assertEqual(
|
||||
round1.semantic_topk(brief_id, entries, k),
|
||||
harness2.semantic_topk(brief_id, entries, k),
|
||||
)
|
||||
if not semantic:
|
||||
self.skipTest(f"lexical parity checked; {round1.VECTORS} absent")
|
||||
|
||||
def test_retriever_k_counts_and_prefix_stability(self):
|
||||
lexical = {
|
||||
|
||||
Reference in New Issue
Block a user