@@ -1320,13 +1322,11 @@ def _row(label: str, values: list[str], p: str | None) -> str:
rows.append(_row("Failed", [str(result.variant_aggregates[vid].tasks_failed) for vid in result.variant_ids], None))
rows.append(_row("Errors", [str(result.variant_aggregates[vid].tasks_error) for vid in result.variant_ids], None))
- def _success_rate(vid: str) -> str:
- agg = result.variant_aggregates[vid]
- evaluable = agg.tasks_run - agg.tasks_error
- rate = (agg.tasks_succeeded / evaluable * 100) if evaluable > 0 else 0.0
- return f"{rate:.1f}%"
+ def _pass_rate(vid: str) -> str:
+ rate = result.variant_aggregates[vid].pass_rate
+ return f"{rate * 100:.1f}%" if rate is not None else "n/a"
- rows.append(_row("Success Rate", [_success_rate(vid) for vid in result.variant_ids], None))
+ rows.append(_row("Pass Rate", [_pass_rate(vid) for vid in result.variant_ids], None))
rows.append(
_row(
diff --git a/tests/_fixtures/report_snapshots/experiment_2variant.md b/tests/_fixtures/report_snapshots/experiment_2variant.md
index e19649fd..2cf11a74 100644
--- a/tests/_fixtures/report_snapshots/experiment_2variant.md
+++ b/tests/_fixtures/report_snapshots/experiment_2variant.md
@@ -19,7 +19,7 @@
| - Token budget | 1 | 0 | — |
| - Cost budget | 0 | 1 | — |
| Errors | 0 | 0 | — |
-| Success Rate | 50.0% | 50.0% | — |
+| Pass Rate | 50.0% | 50.0% | — |
| Score | 0.700 ± 0.283 | 0.825 ± 0.177 | 0.658 |
| Avg Duration (s) | 35.0 ± 7.1 | 55.0 ± 7.1 | 0.106 |
| Assistant Turns | 5.5 ± 0.7 | 7.5 ± 0.7 | 0.106 |
diff --git a/tests/_fixtures/report_snapshots/experiment_3variant.md b/tests/_fixtures/report_snapshots/experiment_3variant.md
index ff93b009..40ede665 100644
--- a/tests/_fixtures/report_snapshots/experiment_3variant.md
+++ b/tests/_fixtures/report_snapshots/experiment_3variant.md
@@ -12,7 +12,7 @@
| Succeeded | 1 | 1 | 1 |
| Failed | 1 | 1 | 1 |
| Errors | 0 | 0 | 0 |
-| Success Rate | 50.0% | 50.0% | 50.0% |
+| Pass Rate | 50.0% | 50.0% | 50.0% |
| Score | 0.650 ± 0.354 | 0.775 ± 0.247 | 0.650 ± 0.212 |
| Avg Duration (s) | 21.0 ± 1.4 | 26.5 ± 2.1 | 31.5 ± 2.1 |
| Assistant Turns | 3.5 ± 0.7 | 6.0 ± 1.4 | 5.5 ± 0.7 |
diff --git a/tests/_fixtures/report_snapshots/experiment_replicates.md b/tests/_fixtures/report_snapshots/experiment_replicates.md
index dfcd8988..73333d07 100644
--- a/tests/_fixtures/report_snapshots/experiment_replicates.md
+++ b/tests/_fixtures/report_snapshots/experiment_replicates.md
@@ -12,7 +12,7 @@
| Succeeded | 1 | 1 | — |
| Failed | 0 | 0 | — |
| Errors | 0 | 0 | — |
-| Success Rate | 100.0% | 100.0% | — |
+| Pass Rate | 100.0% | 100.0% | — |
| Score | 0.900 | 0.650 | — |
| Avg Duration (s) | 3.3 | 3.3 | — |
| Tokens | 1,000 | 1,000 | — |
diff --git a/tests/_fixtures/report_snapshots/run_full.md b/tests/_fixtures/report_snapshots/run_full.md
index 1b9dc828..3e14b5ec 100644
--- a/tests/_fixtures/report_snapshots/run_full.md
+++ b/tests/_fixtures/report_snapshots/run_full.md
@@ -11,7 +11,7 @@
- **Succeeded**: 1
- **Failed**: 1 (incl. 1 token budget, 0 cost budget exceeded)
- **Errors**: 0
-- **Success Rate**: 50.0%
+- **Pass Rate**: 50.0% (1/2)
- **Avg Reliability Score**: 0.625
- **Avg Generation Latency**: 10.2s
- **Total Assistant Turns**: 4
diff --git a/tests/_fixtures/report_snapshots/run_minimal.md b/tests/_fixtures/report_snapshots/run_minimal.md
index 87969535..be0e8308 100644
--- a/tests/_fixtures/report_snapshots/run_minimal.md
+++ b/tests/_fixtures/report_snapshots/run_minimal.md
@@ -10,7 +10,7 @@
- **Succeeded**: 0
- **Failed**: 0
- **Errors**: 0
-- **Success Rate**: 0.0%
+- **Pass Rate**: n/a (0/0)
## Task Details
diff --git a/tests/test_agent_timeout.py b/tests/test_agent_timeout.py
index bd9fff7b..5777dea0 100644
--- a/tests/test_agent_timeout.py
+++ b/tests/test_agent_timeout.py
@@ -258,3 +258,60 @@ async def mock_query(prompt, options):
):
await agent.communicate("prompt") # no timeout
mock_transport_cls.assert_not_called()
+
+
+class _MockTextBlock:
+ def __init__(self, text: str) -> None:
+ self.text = text
+
+
+class _MockAssistantMessage:
+ """Duck-typed SDK AssistantMessage carrying one text block and a usage dict."""
+
+ def __init__(self, text: str, usage: dict[str, int]) -> None:
+ self.content = [_MockTextBlock(text)]
+ self.model = "claude-sonnet-5"
+ self.usage = usage
+ self.stop_reason = "end_turn"
+ self.message_id = "msg_partial_1"
+
+
+@pytest.mark.asyncio
+async def test_external_cancel_parks_the_turn_it_interrupted():
+ """A turn cancelled from outside must leave its telemetry on ``pending_turn``.
+
+ The task-timeout kill path: the turn never returns a record and the frame that
+ held one unwinds, so the pending slot is the only place its spend can survive.
+ """
+ config = parse_agent_config(type=AgentKind.CLAUDE_CODE, permission_mode="acceptEdits", model="claude-sonnet-5")
+ agent = ClaudeCodeAgent(config)
+
+ with tempfile.TemporaryDirectory() as tmpdir:
+ await agent.start(tmpdir)
+
+ streaming = asyncio.Event()
+
+ async def mock_query(prompt, options):
+ yield _MockAssistantMessage("working on it", {"input_tokens": 40_000, "output_tokens": 2_000})
+ streaming.set()
+ # Still mid-turn when the cancel lands, as on a real hard kill: no
+ # terminal ResultMessage is ever delivered.
+ await asyncio.sleep(30)
+
+ with patch("coder_eval.agents.claude_code_agent.query", mock_query):
+ turn = asyncio.create_task(agent.communicate("prompt"))
+ await streaming.wait()
+ turn.cancel()
+ with pytest.raises(asyncio.CancelledError):
+ await asyncio.wait_for(turn, timeout=5)
+
+ partial = agent.pending_turn
+ assert partial is not None, "the interrupted turn's record was discarded"
+ assert partial.crashed is True
+ usage = partial.token_usage
+ assert usage is not None
+ assert usage.output_tokens == 2_000
+ # No ResultMessage means the backend never priced this turn, so the cost is
+ # backfilled from the buckets at the model's rate.
+ assert usage.total_cost_usd is not None
+ assert usage.total_cost_usd > 0
diff --git a/tests/test_antigravity_agent.py b/tests/test_antigravity_agent.py
index 4ac68408..99747950 100644
--- a/tests/test_antigravity_agent.py
+++ b/tests/test_antigravity_agent.py
@@ -202,7 +202,11 @@ def test_tool_name_map_covers_core_builtins():
"gemini-3.1-pro-preview",
"gemini-3.1-pro-preview-customtools",
"gemini-3-pro-preview",
+ "gemini-3.6-flash",
"gemini-3.5-flash",
+ "gemini-3.5-flash-lite",
+ "gemini-3.1-flash-lite",
+ "gemini-3.1-flash-lite-preview",
"gemini-3-flash-preview",
],
)
diff --git a/tests/test_cost_accounting_paths.py b/tests/test_cost_accounting_paths.py
new file mode 100644
index 00000000..9193f7dc
--- /dev/null
+++ b/tests/test_cost_accounting_paths.py
@@ -0,0 +1,362 @@
+"""Cost accounting on the error and timeout paths, where spend went missing.
+
+Two seams: the ``check_pricing_coverage`` pre-flight, which warns when a model the
+run will use has no rate, and the ``eval_result_to_task_dict`` row projection,
+which reports judge/simulator spend and flags a row whose costs are only partial.
+"""
+
+from __future__ import annotations
+
+import logging
+from datetime import datetime
+from pathlib import Path
+from typing import Any
+
+import pytest
+
+from coder_eval.models import (
+ ClaudeCodeAgentConfig,
+ EvaluationResult,
+ FinalStatus,
+ JudgeCriterionResult,
+ ResolvedTask,
+ SimulationTelemetry,
+ TaskDefinition,
+ TokenUsage,
+ TurnRecord,
+)
+from coder_eval.orchestration.batch import check_pricing_coverage
+from coder_eval.reports_experiment import eval_result_to_task_dict
+
+
+def _resolved(model: str | None, tmp_path: Path, criteria: list[dict[str, Any]] | None = None) -> ResolvedTask:
+ task = TaskDefinition(
+ task_id="t1",
+ description="d",
+ initial_prompt="p",
+ agent=ClaudeCodeAgentConfig(type="claude-code", model=model),
+ sandbox={"driver": "tempdir"},
+ success_criteria=criteria or [{"type": "file_exists", "path": "f.py", "description": "d"}],
+ )
+ return ResolvedTask(
+ task=task,
+ task_file=tmp_path / "t1.yaml",
+ run_dir=tmp_path / "runs" / "t1",
+ variant_id="default",
+ )
+
+
+def _turn(iteration: int, usage: TokenUsage | None, model: str | None = None, crashed: bool = False) -> TurnRecord:
+ return TurnRecord(
+ iteration=iteration,
+ user_input="p",
+ agent_output="o",
+ token_usage=usage,
+ model_used=model,
+ crashed=crashed,
+ )
+
+
+def _result(turns: list[TurnRecord], *, model: str | None = "claude-sonnet-5", **extra) -> EvaluationResult:
+ return EvaluationResult(
+ task_id="t1",
+ task_description="d",
+ agent_type="claude-code",
+ model_used=model,
+ started_at=datetime(2026, 7, 28, 0, 0, 0),
+ final_status=FinalStatus.SUCCESS,
+ iteration_count=len(turns),
+ iterations=turns,
+ **extra,
+ )
+
+
+class TestPricingPreflight:
+ def test_priced_model_is_silent(self, tmp_path, caplog):
+ with caplog.at_level(logging.WARNING):
+ assert check_pricing_coverage([_resolved("claude-sonnet-5", tmp_path)]) == []
+ assert "No pricing rate" not in caplog.text
+
+ def test_unpriced_model_warns_but_runs(self, tmp_path, caplog):
+ """Default is a loud warning, not a refusal: a brand-new model stays evaluable."""
+ with caplog.at_level(logging.WARNING):
+ missing = check_pricing_coverage([_resolved("claude-sonnet-99", tmp_path)])
+ assert missing == ["claude-sonnet-99"]
+ assert "No pricing rate" in caplog.text
+ assert "understate the bill" in caplog.text
+
+ def test_unpinned_model_is_not_flagged(self, tmp_path):
+ """A task deferring its model to the route can't be pre-flighted from here."""
+ assert check_pricing_coverage([_resolved(None, tmp_path)]) == []
+
+ def test_judge_criterion_model_is_pre_flighted(self, tmp_path):
+ """A judge call is ALWAYS priced from the rate card, so its model matters most.
+
+ No judge backend reports a cost, unlike an agent whose SDK prices a clean
+ turn, so an unpriced ``criterion.model`` loses money on every graded row.
+ """
+ judge = {"type": "llm_judge", "description": "d", "prompt": "grade it", "model": "claude-sonnet-99"}
+ assert check_pricing_coverage([_resolved("claude-sonnet-5", tmp_path, [judge])]) == ["claude-sonnet-99"]
+
+ def test_default_judge_model_is_priced(self, tmp_path):
+ """The out-of-the-box judge must not warn on a stock task."""
+ judge = {"type": "llm_judge", "description": "d", "prompt": "grade it"}
+ assert check_pricing_coverage([_resolved("claude-sonnet-5", tmp_path, [judge])]) == []
+
+
+class TestRowCostProjection:
+ def test_cost_complete_false_when_a_turn_is_unpriced(self):
+ result = _result(
+ [
+ _turn(1, TokenUsage(uncached_input_tokens=10, output_tokens=1, total_cost_usd=0.1)),
+ _turn(2, TokenUsage(uncached_input_tokens=10, output_tokens=1)),
+ ]
+ )
+ assert eval_result_to_task_dict(result)["cost_complete"] is False
+
+ def test_cost_complete_true_when_every_burning_turn_is_priced(self):
+ result = _result([_turn(1, TokenUsage(uncached_input_tokens=10, output_tokens=1, total_cost_usd=0.1))])
+ assert eval_result_to_task_dict(result)["cost_complete"] is True
+
+ def test_cost_complete_true_when_nothing_burned(self):
+ """An error before the agent ran genuinely cost nothing — not 'missing cost'."""
+ assert eval_result_to_task_dict(_result([]))["cost_complete"] is True
+ assert eval_result_to_task_dict(_result([_turn(1, TokenUsage())]))["cost_complete"] is True
+
+ def test_cost_complete_false_when_a_task_timeout_preserved_no_turn(self):
+ """A task timeout with zero turns is unrecorded spend, not free.
+
+ The watchdog SIGKILLs the agent by PID, so unlike a turn-level timeout this
+ never reaches ``_on_attempt_failure`` and no partial turn is drained. The
+ row lands with no turns, no tokens and no cost. Since a task timeout means
+ the evaluation loop was still running, calling that row fully priced would
+ be a false claim — the one case where cost is missing with no tokens to
+ point at.
+ """
+ result = _result([])
+ result.final_status = FinalStatus.TIMEOUT
+ assert eval_result_to_task_dict(result)["cost_complete"] is False
+
+ def test_recovered_timeout_row_reports_its_cost_and_still_flags_incomplete(self):
+ """Recovering the killed turn narrows the gap; it does not close it.
+
+ The interrupted turn is drained onto the row and priced, so the row carries
+ a real number instead of nothing. The generation the agent was waiting on
+ when it died was never delivered by anyone, so the number is still a floor.
+ """
+ killed = _turn(1, TokenUsage(uncached_input_tokens=40_000, output_tokens=2_000, total_cost_usd=0.15))
+ killed.crashed = True
+ result = _result([killed])
+ result.final_status = FinalStatus.TIMEOUT
+ result.total_token_usage = TokenUsage(uncached_input_tokens=40_000, output_tokens=2_000, total_cost_usd=0.15)
+
+ row = eval_result_to_task_dict(result)
+ assert row["total_cost_usd"] == pytest.approx(0.15)
+ assert row["cost_complete"] is False
+
+ def test_cost_complete_true_for_a_fast_error_with_no_turn(self):
+ """The companion: a setup failure has no turns either, and really is free.
+
+ Keyed on status rather than elapsed time so a slow setup failure stays free
+ while a task timeout never does.
+ """
+ result = _result([])
+ result.final_status = FinalStatus.ERROR
+ assert eval_result_to_task_dict(result)["cost_complete"] is True
+
+ def test_judge_cost_rolls_up_onto_the_row(self):
+ """Judge spend was captured per criterion and totalled nowhere."""
+ result = _result([_turn(1, TokenUsage(uncached_input_tokens=10, output_tokens=1, total_cost_usd=0.1))])
+ result.total_token_usage = TokenUsage(uncached_input_tokens=10, output_tokens=1, total_cost_usd=0.1)
+ result.success_criteria_results = [
+ JudgeCriterionResult(
+ criterion_type="llm_judge",
+ description="d",
+ score=1.0,
+ token_usage=TokenUsage(uncached_input_tokens=5000, output_tokens=500, total_cost_usd=0.02),
+ ),
+ JudgeCriterionResult(
+ criterion_type="llm_judge",
+ description="d2",
+ score=1.0,
+ token_usage=TokenUsage(uncached_input_tokens=5000, output_tokens=500, total_cost_usd=0.03),
+ ),
+ ]
+ row = eval_result_to_task_dict(result)
+ assert row["judge_cost_usd"] == pytest.approx(0.05)
+ # Folded into the row's bill, and kept out of the agent-only slice.
+ assert row["total_cost_usd"] == pytest.approx(0.15)
+ assert row["agent_cost_usd"] == pytest.approx(0.1)
+
+ def test_no_judge_means_no_judge_cost(self):
+ """None, not 0.0 — 'no judge ran' must stay distinct from 'a judge ran free'."""
+ result = _result([_turn(1, TokenUsage(uncached_input_tokens=10, output_tokens=1, total_cost_usd=0.1))])
+ assert eval_result_to_task_dict(result)["judge_cost_usd"] is None
+
+ def test_unpriced_judge_is_skipped_not_fatal(self):
+ """An unpriced judge lowers the total; it must never raise or zero the row.
+
+ The pre-flight warns about this at dispatch. After that the run carries on
+ and the figures are a floor, which is the trade this framework makes
+ everywhere: cost degrades, the evaluation does not.
+ """
+ result = _result([_turn(1, TokenUsage(uncached_input_tokens=10, output_tokens=1, total_cost_usd=0.1))])
+ result.total_token_usage = TokenUsage(uncached_input_tokens=10, output_tokens=1, total_cost_usd=0.1)
+ result.success_criteria_results = [
+ JudgeCriterionResult(
+ criterion_type="llm_judge",
+ description="priced",
+ score=1.0,
+ token_usage=TokenUsage(uncached_input_tokens=5000, output_tokens=500, total_cost_usd=0.02),
+ ),
+ JudgeCriterionResult(
+ criterion_type="llm_judge",
+ description="unpriced",
+ score=1.0,
+ token_usage=TokenUsage(uncached_input_tokens=5000, output_tokens=500),
+ ),
+ ]
+ row = eval_result_to_task_dict(result)
+ assert row["judge_cost_usd"] == pytest.approx(0.02)
+ assert row["total_cost_usd"] == pytest.approx(0.12)
+
+ @staticmethod
+ def _simulated(**env) -> EvaluationResult:
+ return _result(
+ [_turn(1, TokenUsage(uncached_input_tokens=10, output_tokens=1, total_cost_usd=0.1))],
+ simulation=SimulationTelemetry(
+ n_trials=1,
+ replicate_index=0,
+ stop_reason="stop_token",
+ simulator_input_tokens=1_000_000,
+ simulator_output_tokens=100_000,
+ total_turns=3,
+ ),
+ environment_info=env,
+ )
+
+ def test_simulator_prices_at_the_route_model_not_the_subject(self):
+ """UserSimulator pins model=None, so it bills at BEDROCK_MODEL.
+
+ A task that pins ``agent.model`` would otherwise mis-bill every simulated
+ row. Here the subject is sonnet-5 ($3/$15) while the route is haiku-4.5
+ ($1/$5), so the simulator must cost the haiku rate.
+ """
+ result = self._simulated(bedrock_model="claude-haiku-4-5-20251001")
+ # 1M uncached input at $1/MTok + 100K output at $5/MTok.
+ assert eval_result_to_task_dict(result)["simulator_cost_usd"] == pytest.approx(1.00 + 0.50)
+
+ def test_simulator_falls_back_to_the_subject_model_off_bedrock(self):
+ """A non-Bedrock route names no model on the record; the subject's is the best available."""
+ result = self._simulated()
+ # sonnet-5 at $3/$15 per MTok.
+ assert eval_result_to_task_dict(result)["simulator_cost_usd"] == pytest.approx(3.0 + 1.5)
+
+ def test_single_shot_row_has_no_simulator_cost(self):
+ result = _result([_turn(1, TokenUsage(uncached_input_tokens=10, output_tokens=1, total_cost_usd=0.1))])
+ assert eval_result_to_task_dict(result)["simulator_cost_usd"] is None
+
+ def test_unpriced_simulator_route_is_skipped_not_fatal(self):
+ """An unpriced simulator route leaves the rest of the row intact."""
+ result = self._simulated(bedrock_model="claude-sonnet-99")
+ result.total_token_usage = TokenUsage(uncached_input_tokens=10, output_tokens=1, total_cost_usd=0.1)
+ row = eval_result_to_task_dict(result)
+ assert row["simulator_cost_usd"] is None
+ # The agent's own spend still lands; only the simulator slice is missing.
+ assert row["total_cost_usd"] == pytest.approx(0.1)
+
+
+class TestTotalCost:
+ """``total_cost_usd`` is the whole bill: agent + judge + simulator."""
+
+ def test_sums_every_component(self):
+ result = _result(
+ [_turn(1, TokenUsage(uncached_input_tokens=10, output_tokens=1, total_cost_usd=1.0))],
+ simulation=SimulationTelemetry(
+ n_trials=1,
+ replicate_index=0,
+ stop_reason="stop_token",
+ simulator_input_tokens=1_000_000,
+ simulator_output_tokens=0,
+ total_turns=2,
+ ),
+ environment_info={"bedrock_model": "claude-haiku-4-5-20251001"},
+ )
+ result.total_token_usage = TokenUsage(uncached_input_tokens=10, output_tokens=1, total_cost_usd=1.0)
+ result.success_criteria_results = [
+ JudgeCriterionResult(
+ criterion_type="llm_judge",
+ description="d",
+ score=1.0,
+ token_usage=TokenUsage(uncached_input_tokens=5000, output_tokens=500, total_cost_usd=0.25),
+ ),
+ ]
+ row = eval_result_to_task_dict(result)
+ # 1.0 agent + 0.25 judge + 1M input at haiku's $1/MTok.
+ assert row["total_cost_usd"] == pytest.approx(2.25)
+ # The agent slice stays broken out so harnesses stay comparable.
+ assert row["agent_cost_usd"] == pytest.approx(1.0)
+
+ def test_none_when_nothing_was_priced(self):
+ """Not 0.0 — an unpriced row must not read as a free one."""
+ assert eval_result_to_task_dict(_result([]))["total_cost_usd"] is None
+
+ def test_run_level_total_is_the_sum_of_both_halves(self):
+ from coder_eval.models import RunSummary
+
+ summary = RunSummary(
+ run_id="2026-07-29_00-00-00",
+ start_time=datetime(2026, 7, 29, 0, 0, 0),
+ end_time=datetime(2026, 7, 29, 1, 0, 0),
+ total_duration_seconds=3600.0,
+ tasks_run=1,
+ tasks_succeeded=1,
+ tasks_failed=0,
+ tasks_error=0,
+ task_results=[
+ {
+ "task_id": "t",
+ "total_cost_usd": 1.25,
+ "agent_cost_usd": 1.0,
+ "judge_cost_usd": 0.2,
+ "simulator_cost_usd": 0.05,
+ },
+ ],
+ framework_version="0.0.0-test",
+ )
+ assert summary.agent_cost_usd == pytest.approx(1.0)
+ assert summary.eval_overhead_cost_usd == pytest.approx(0.25)
+ assert summary.total_cost_usd == pytest.approx(1.25)
+ assert summary.model_dump()["total_cost_usd"] == pytest.approx(1.25)
+
+
+class TestErrorDiagnosticsOnTheRow:
+ """Errors count as misses, so the rollup has to say why it lost those points.
+
+ Without these, a run's zero-iteration errors cannot be characterised from
+ run.json at all: every one needs its own task.json fetch.
+ """
+
+ def test_error_message_and_category_land_on_the_row(self):
+ result = _result([], model=None)
+ result.final_status = FinalStatus.ERROR
+ result.error_message = "sandbox setup failed: no space left on device"
+ result.error_details = {"error_category": "disk_full", "component": "orchestrator.setup"}
+
+ row = eval_result_to_task_dict(result)
+ assert row["error_message"] == "sandbox setup failed: no space left on device"
+ assert row["error_category"] == "disk_full"
+
+ def test_error_message_is_truncated(self):
+ result = _result([], model=None)
+ result.final_status = FinalStatus.ERROR
+ result.error_message = "x" * 5000
+
+ row = eval_result_to_task_dict(result)
+ assert len(row["error_message"]) < 500
+ assert row["error_message"].endswith("…")
+
+ def test_clean_row_carries_no_error_fields(self):
+ row = eval_result_to_task_dict(_result([]))
+ assert row["error_message"] is None
+ assert row["error_category"] is None
diff --git a/tests/test_experiment_reports.py b/tests/test_experiment_reports.py
index de432278..3e212da5 100644
--- a/tests/test_experiment_reports.py
+++ b/tests/test_experiment_reports.py
@@ -529,7 +529,7 @@ def test_variant_report_content(self, tmp_path, sample_result):
# New: variant report should include experiment context and summary
assert "**Experiment**: model-comparison" in content
assert "## Summary" in content
- assert "Success Rate" in content
+ assert "Pass Rate" in content
def test_variant_html_task_links_include_replicate_segment(self, tmp_path, sample_result):
"""variant.html task links must include the /00/ replicate segment so they
@@ -641,7 +641,7 @@ def test_experiment_report_has_aggregate_metrics(self):
assert "| Avg Duration (s) |" in md
assert "| Tokens |" in md
assert "| Tasks Run |" in md
- assert "| Success Rate |" in md
+ assert "| Pass Rate |" in md
# Two-variant comparison should show p-value column
assert "p-value" in md
diff --git a/tests/test_reports.py b/tests/test_reports.py
index 8b741ddb..756a345f 100644
--- a/tests/test_reports.py
+++ b/tests/test_reports.py
@@ -212,7 +212,9 @@ def test_generate_markdown_basic():
assert "Succeeded**: 2" in report_md
assert "Failed**: 1" in report_md
assert "Errors**: 0" in report_md
- assert "Success Rate**: 66.7%" in report_md
+ assert "Pass Rate**: 66.7% (2/3)" in report_md
+ # No errors in this run, so no error-share line to explain the denominator.
+ assert "Error Share" not in report_md
# Check P0 aggregate metrics
assert "Avg Reliability Score**:" in report_md
@@ -259,7 +261,8 @@ def test_generate_markdown_empty_tasks():
report_md = ReportGenerator.generate_markdown(summary)
assert "Total Tasks**: 0" in report_md
- assert "Success Rate**: 0.0%" in report_md
+ # A run with no tasks has no pass rate; 0.0% would read as "everything failed".
+ assert "Pass Rate**: n/a (0/0)" in report_md
# No aggregate metrics when there are no tasks
assert "Avg Reliability Score" not in report_md
assert "Generation Metrics" not in report_md
diff --git a/tests/test_route_seam_exhaustiveness.py b/tests/test_route_seam_exhaustiveness.py
index 45265fb1..b58a1307 100644
--- a/tests/test_route_seam_exhaustiveness.py
+++ b/tests/test_route_seam_exhaustiveness.py
@@ -72,7 +72,7 @@ async def _stub_anthropic(**_: object) -> dict[str, object]:
monkeypatch.setattr(llm_judge, "invoke_bedrock_judge_async", _stub_bedrock)
monkeypatch.setattr(llm_judge, "invoke_anthropic_judge_async", _stub_anthropic)
monkeypatch.setattr(llm_judge, "extract_verdict_from_anthropic_response", lambda _resp: (None, "stub"))
- monkeypatch.setattr(llm_judge, "token_usage_from_anthropic_dict", lambda _resp: None)
+ monkeypatch.setattr(llm_judge, "token_usage_from_anthropic_dict", lambda _resp, **_kwargs: None)
criterion = MagicMock()
for r in _INSTANCES:
result = await _invoke_tool_channel(criterion=criterion, route=r, system_msg="s", user_msg="u") # type: ignore[arg-type]
diff --git a/tests/test_run_metrics.py b/tests/test_run_metrics.py
new file mode 100644
index 00000000..6ac5baff
--- /dev/null
+++ b/tests/test_run_metrics.py
@@ -0,0 +1,253 @@
+"""Run-level derived metrics: one pass-rate denominator, and honest cost totals.
+
+Two bugs are pinned here.
+
+**The denominator.** ``pass_rate`` used to be ``succeeded / (run - error)``, which
+paid a bonus for erroring: the more a run fell over, the smaller its denominator
+got, up to the degenerate case of a run rendering as a perfect score while passing
+a handful of rows. Every surface now divides by ``tasks_run``.
+
+**The bill.** Cost was summed over whatever rows happened to carry one, so a run
+whose model was missing from the rate card, or whose turns were killed before the
+backend reported a cost, understated its spend silently. Unpriced spend is now
+counted and the total is labelled a floor.
+"""
+
+from __future__ import annotations
+
+from datetime import datetime
+from pathlib import Path
+from typing import Any
+
+import pytest
+import yaml
+
+from coder_eval.models import FinalStatus, RunSummary, sum_costs
+from coder_eval.pricing import is_priced, unpriced_models
+
+
+def _row(status: FinalStatus | str, **extra: Any) -> dict[str, Any]:
+ """A run.json row. ``total_cost_usd`` is derived the way the real projection derives it.
+
+ Callers give the three cost slices; the row's total is their sum, so a test can
+ never assert against a total the projection would not have written.
+ """
+ row: dict[str, Any] = {"task_id": f"t{id(extra)}", "status": status, **extra}
+ if any(k in extra for k in ("agent_cost_usd", "judge_cost_usd", "simulator_cost_usd")):
+ row["total_cost_usd"] = sum_costs(
+ extra.get("agent_cost_usd"), extra.get("judge_cost_usd"), extra.get("simulator_cost_usd")
+ )
+ return row
+
+
+def _summary(rows: list[dict[str, Any]]) -> RunSummary:
+ """Build a RunSummary whose stored counts are derived from ``rows``."""
+ statuses = [FinalStatus(r["status"]) for r in rows]
+ return RunSummary(
+ run_id="2026-07-28_00-00-00",
+ start_time=datetime(2026, 7, 28, 0, 0, 0),
+ end_time=datetime(2026, 7, 28, 1, 0, 0),
+ total_duration_seconds=3600.0,
+ tasks_run=len(rows),
+ tasks_succeeded=sum(1 for s in statuses if s.category == "succeeded"),
+ tasks_failed=sum(1 for s in statuses if s.category == "failed"),
+ tasks_error=sum(1 for s in statuses if s.category == "error"),
+ task_results=rows,
+ framework_version="0.0.0-test",
+ )
+
+
+class TestPassRateDenominator:
+ def test_errors_count_as_misses(self):
+ """The whole point: an error is in the denominator, not excluded from it."""
+ summary = _summary(
+ [
+ _row(FinalStatus.SUCCESS),
+ _row(FinalStatus.FAILURE),
+ _row(FinalStatus.ERROR),
+ _row(FinalStatus.ERROR),
+ ]
+ )
+ assert summary.pass_rate == pytest.approx(0.25)
+ # The old formula would have divided by 2 and published 50%.
+ assert summary.error_share == pytest.approx(0.5)
+
+ def test_build_failed_is_an_error_row(self):
+ """BUILD_FAILED buckets to 'error' and so still lands in the denominator."""
+ summary = _summary([_row(FinalStatus.SUCCESS), _row(FinalStatus.BUILD_FAILED)])
+ assert summary.tasks_error == 1
+ assert summary.pass_rate == pytest.approx(0.5)
+
+ def test_timeout_and_budget_statuses_are_failures_not_errors(self):
+ """These are task outcomes; they were always in the denominator and stay there."""
+ summary = _summary(
+ [
+ _row(FinalStatus.SUCCESS),
+ _row(FinalStatus.TIMEOUT),
+ _row(FinalStatus.MAX_TURNS_EXHAUSTED),
+ _row(FinalStatus.TOKEN_BUDGET_EXCEEDED),
+ _row(FinalStatus.COST_BUDGET_EXCEEDED),
+ ]
+ )
+ assert summary.tasks_error == 0
+ assert summary.error_share == pytest.approx(0.0)
+ assert summary.pass_rate == pytest.approx(0.2)
+
+ def test_mostly_errored_run_cannot_render_as_perfect(self):
+ """The reductio: 854 errors out of 861 rows.
+
+ Under the old formula this reported 100.0% (7 evaluable, 7 passed). It must
+ read as what it was.
+ """
+ rows = [_row(FinalStatus.SUCCESS) for _ in range(7)] + [_row(FinalStatus.ERROR) for _ in range(854)]
+ summary = _summary(rows)
+ assert summary.pass_rate == pytest.approx(7 / 861)
+ assert summary.pass_rate < 0.01
+
+ def test_empty_run_has_no_rate(self):
+ """0/0 is unknown, not 0% — which would read as 'everything failed'."""
+ summary = _summary([])
+ assert summary.pass_rate is None
+ assert summary.error_share is None
+
+ def test_variant_aggregate_shares_the_denominator(self):
+ """An A/B whose variants error at different rates must compare like with like."""
+ from coder_eval.models import VariantAggregate
+
+ agg = VariantAggregate(
+ variant_id="v",
+ tasks_run=4,
+ tasks_succeeded=1,
+ tasks_failed=1,
+ tasks_error=2,
+ average_score=0.5,
+ average_duration=1.0,
+ )
+ assert agg.pass_rate == pytest.approx(0.25)
+
+ def test_rates_serialize_into_run_json(self):
+ """Downstream consumers must be able to READ the rate instead of re-deriving it.
+
+ Independent re-derivations drift, and then two consumers publish different
+ rates for the same run.
+ """
+ summary = _summary([_row(FinalStatus.SUCCESS), _row(FinalStatus.ERROR)])
+ dumped = summary.model_dump()
+ assert dumped["pass_rate"] == pytest.approx(0.5)
+ assert dumped["error_share"] == pytest.approx(0.5)
+ # And the round-trip tolerates its own output (computed fields aren't inputs).
+ assert RunSummary.model_validate_json(summary.model_dump_json()).pass_rate == pytest.approx(0.5)
+
+
+class TestCostCompleteness:
+ def test_unpriced_row_is_counted_and_flagged(self):
+ summary = _summary(
+ [
+ _row(FinalStatus.SUCCESS, total_tokens=1000, agent_cost_usd=0.5, cost_complete=True),
+ _row(FinalStatus.TIMEOUT, total_tokens=8_000_000, agent_cost_usd=None, cost_complete=False),
+ ]
+ )
+ assert summary.tasks_cost_incomplete == 1
+ assert summary.cost_complete is False
+ # The priced row's cost still totals — a floor beats no number at all.
+ assert summary.agent_cost_usd == pytest.approx(0.5)
+
+ def test_partially_priced_row_is_counted(self):
+ """A row whose cost is a partial sum of its own turns is incomplete.
+
+ This is the timeout shape: turn 1 priced by the SDK, turn 2 killed
+ mid-flight with real tokens and no cost. Summing turn 1 alone and calling
+ it the row's cost is the silent 19% understatement.
+ """
+ summary = _summary(
+ [_row(FinalStatus.TIMEOUT, total_tokens=5_000_000, agent_cost_usd=1.25, cost_complete=False)]
+ )
+ assert summary.tasks_cost_incomplete == 1
+ assert summary.cost_complete is False
+
+ def test_zero_token_error_is_not_unpriced(self):
+ """A row that died before the agent ran genuinely cost nothing."""
+ summary = _summary([_row(FinalStatus.ERROR, total_tokens=0, agent_cost_usd=None, cost_complete=True)])
+ assert summary.tasks_cost_incomplete == 0
+ assert summary.cost_complete is True
+ assert summary.agent_cost_usd is None
+
+ def test_eval_overhead_is_separate_from_the_agent_bill(self):
+ """Judge/simulator spend is real money but must not inflate the agent's cost.
+
+ It is a property of the suite's criteria and identical across harnesses, so
+ folding it in would make two harnesses look closer than they are.
+ """
+ summary = _summary(
+ [
+ _row(
+ FinalStatus.SUCCESS,
+ total_tokens=1000,
+ agent_cost_usd=1.0,
+ cost_complete=True,
+ judge_cost_usd=0.2,
+ simulator_cost_usd=0.05,
+ ),
+ ]
+ )
+ assert summary.agent_cost_usd == pytest.approx(1.0)
+ assert summary.eval_overhead_cost_usd == pytest.approx(0.25)
+ # And the bill is the two together, which is what `total_cost_usd` means.
+ assert summary.total_cost_usd == pytest.approx(1.25)
+
+ def test_no_overhead_reports_none(self):
+ summary = _summary([_row(FinalStatus.SUCCESS, total_tokens=1, agent_cost_usd=0.1, cost_complete=True)])
+ assert summary.eval_overhead_cost_usd is None
+
+
+class TestPricingCoverage:
+ def test_is_priced_normalizes_bedrock_prefixes(self):
+ assert is_priced("claude-sonnet-5")
+ assert is_priced("eu.anthropic.claude-sonnet-5")
+ assert not is_priced("claude-sonnet-99")
+
+ def test_unpriced_models_dedupes_and_drops_empty(self):
+ assert unpriced_models(["claude-sonnet-5", None, "", "made-up-1", "made-up-1", "made-up-0"]) == [
+ "made-up-0",
+ "made-up-1",
+ ]
+
+
+class TestRateCardCoversCheckedInExperiments:
+ """CI guard: a model referenced by a committed experiment must be priced.
+
+ The rate card is a static table baked into the installed version, so a model
+ whose rates land in a later release prices as null on any turn the agent's own
+ backend did not price. This class is what makes the next new model fail CI
+ instead of a run's cost column.
+ """
+
+ @staticmethod
+ def _models_in(doc: dict[str, Any]) -> set[str]:
+ found: set[str] = set()
+ for block in (doc.get("defaults") or {}, *(doc.get("variants") or [])):
+ if not isinstance(block, dict):
+ continue
+ agent = block.get("agent")
+ if isinstance(agent, dict) and agent.get("model"):
+ found.add(str(agent["model"]))
+ return found
+
+ def test_every_experiment_model_is_priced(self):
+ experiments_dir = Path(__file__).resolve().parents[1] / "experiments"
+ referenced: dict[str, str] = {}
+ for path in sorted(experiments_dir.glob("*.yaml")):
+ doc = yaml.safe_load(path.read_text(encoding="utf-8")) or {}
+ for model in self._models_in(doc):
+ referenced.setdefault(model, path.name)
+
+ # Guard the guard: an empty scan would pass vacuously.
+ assert referenced, f"No agent.model found under {experiments_dir}; the discovery filter is broken."
+
+ missing = {m: src for m, src in referenced.items() if not is_priced(m)}
+ assert not missing, (
+ "Models referenced by committed experiments have no pricing rate: "
+ + ", ".join(f"{m!r} (from {src})" for m, src in sorted(missing.items()))
+ + ". Add the rate to coder_eval.pricing._PRICING — an unpriced model records "
+ + "null cost for every task and understates the run's bill silently."
+ )
diff --git a/tests/test_timeout_orchestrator.py b/tests/test_timeout_orchestrator.py
index 395ffdba..2bca85b1 100644
--- a/tests/test_timeout_orchestrator.py
+++ b/tests/test_timeout_orchestrator.py
@@ -16,6 +16,7 @@
FileExistsCriterion,
SandboxConfig,
TaskDefinition,
+ TokenUsage,
TurnRecord,
)
from coder_eval.orchestrator import Orchestrator
@@ -227,6 +228,87 @@ async def slow_loop():
mock_agent.kill_sync.assert_called_once()
+@pytest.mark.asyncio
+async def test_task_timeout_recovers_the_killed_turn(tmp_path) -> None:
+ """A hard-killed task's spend lands on the result instead of vanishing.
+
+ The agent parks the interrupted turn on ``pending_turn`` when it is cancelled,
+ and the task-timeout handler is the only reader of that slot: the cancel is a
+ BaseException, so it never reaches the retry executor's per-attempt hook that
+ drains it on a turn-level timeout. Without the drain the row reports no turns
+ and no cost for a task that spent real money.
+ """
+ task = _make_task(task_timeout=0.1)
+ run_dir = tmp_path / "run" / "timeout_test"
+ run_dir.mkdir(parents=True)
+
+ orchestrator = Orchestrator(task=task, run_dir=run_dir, variant_id="test-variant")
+ orchestrator._setup = AsyncMock() # type: ignore[method-assign]
+ orchestrator._cleanup = AsyncMock() # type: ignore[method-assign]
+
+ partial = TurnRecord(
+ iteration=1,
+ user_input="test prompt",
+ agent_output="
",
+ crashed=True,
+ token_usage=TokenUsage(uncached_input_tokens=40_000, output_tokens=2_000, total_cost_usd=0.15),
+ )
+
+ mock_agent = MagicMock()
+ mock_agent.pending_turn = partial
+ mock_agent.discard_pending_turn = AsyncMock()
+ mock_agent.get_sdk_options = MagicMock(return_value=None)
+ orchestrator.agent = mock_agent
+
+ async def slow_loop():
+ await asyncio.sleep(10)
+ return False
+
+ orchestrator._evaluation_loop = slow_loop # type: ignore[method-assign]
+
+ result = await orchestrator.run()
+
+ assert result.final_status == "TIMEOUT"
+ assert result.iterations == [partial]
+ assert result.total_token_usage is not None
+ assert result.total_token_usage.output_tokens == 2_000
+ assert result.total_token_usage.total_cost_usd == pytest.approx(0.15)
+ # Drained through the documented contract, so the slot is left clean.
+ mock_agent.discard_pending_turn.assert_awaited_once()
+
+
+@pytest.mark.asyncio
+async def test_task_timeout_with_nothing_to_recover_still_lands(tmp_path) -> None:
+ """A task killed before its first turn has nothing parked, and that is not an error.
+
+ The recovery is best-effort and runs on the way to a saved row, so an empty slot
+ must leave the TIMEOUT row intact rather than raising through teardown.
+ """
+ task = _make_task(task_timeout=0.1)
+ run_dir = tmp_path / "run" / "timeout_test"
+ run_dir.mkdir(parents=True)
+
+ orchestrator = Orchestrator(task=task, run_dir=run_dir, variant_id="test-variant")
+ orchestrator._setup = AsyncMock() # type: ignore[method-assign]
+ orchestrator._cleanup = AsyncMock() # type: ignore[method-assign]
+
+ mock_agent = MagicMock()
+ mock_agent.pending_turn = None
+ mock_agent.get_sdk_options = MagicMock(return_value=None)
+ orchestrator.agent = mock_agent
+
+ async def slow_loop():
+ await asyncio.sleep(10)
+ return False
+
+ orchestrator._evaluation_loop = slow_loop # type: ignore[method-assign]
+
+ result = await orchestrator.run()
+
+ assert result.final_status == "TIMEOUT"
+ assert result.iterations == []
+
+
@pytest.mark.asyncio
async def test_turn_timeout_not_rewrapped_as_task_timeout(tmp_path) -> None:
"""A per-turn TurnTimeoutError propagates unchanged even when a larger
diff --git a/tests/test_token_usage.py b/tests/test_token_usage.py
index 7eb70b72..71bf6035 100644
--- a/tests/test_token_usage.py
+++ b/tests/test_token_usage.py
@@ -436,8 +436,8 @@ def test_timeout_turn_backfills_cost_from_buckets(self):
]
usage = ClaudeCodeAgent._build_token_usage(messages, None, None, None, "claude-opus-4-8")
assert usage is not None
- # opus-4-8: 15/M in, 75/M out, 18.75/M cache-write, 1.50/M cache-read.
- expected = (1_000_000 * 15.0 + 500_000 * 75.0 + 200_000 * 18.75 + 4_000_000 * 1.50) / 1_000_000
+ # opus-4-8: 5/M in, 25/M out, 6.25/M cache-write, 0.50/M cache-read.
+ expected = (1_000_000 * 5.0 + 500_000 * 25.0 + 200_000 * 6.25 + 4_000_000 * 0.50) / 1_000_000
assert usage.total_cost_usd == pytest.approx(expected)
def test_backfill_via_resultmessage_snapshot_path(self):
@@ -875,6 +875,7 @@ def test_token_section_handles_missing_cost(self):
"output_tokens": 2000,
"total_tokens": 5000,
"total_cost_usd": None,
+ "cost_complete": False,
},
]
lines = ReportGenerator._generate_token_usage_section(task_results)
@@ -882,9 +883,92 @@ def test_token_section_handles_missing_cost(self):
assert "## Token Usage" in joined
assert "**Total Tokens**: 5,000 (input: 3,000, output: 2,000)" in joined
- assert "**Total Cost**" not in joined # No cost line when all costs are None
+ # Tokens were burned, so there IS a bill — the report must say the number is
+ # missing rather than omit the cost line, which reads as "this run was free".
+ assert "**Total Cost**: unavailable" in joined
+ assert "1 task(s) have spend missing from this total" in joined
assert "| task1 | 3,000 | 2,000 | 0 | 0 | 5,000 | N/A |" in joined
+ def test_token_section_leaves_a_legacy_row_uncaveated(self):
+ """A row predating ``cost_complete`` is read as priced, not inferred.
+
+ The same shape as above minus the flag, which is what runs written before
+ the field look like. Deliberately silent: inferring unpriced-ness from
+ tokens would give old runs a caveat at the price of a second definition of
+ "unpriced", and every new run carries the flag.
+ """
+ task_results = [
+ {"task_id": "task1", "total_tokens": 5000, "total_cost_usd": None},
+ ]
+ joined = "\n".join(ReportGenerator._generate_token_usage_section(task_results))
+
+ assert "## Token Usage" in joined
+ assert "spend missing" not in joined
+
+ def test_token_section_omits_cost_when_no_tokens_burned(self):
+ """A row that burned nothing is genuinely free — no unpriced warning."""
+ task_results = [
+ {
+ "task_id": "task1",
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "total_tokens": 0,
+ "total_cost_usd": None,
+ },
+ ]
+ joined = "\n".join(ReportGenerator._generate_token_usage_section(task_results))
+
+ assert "## Token Usage" in joined
+ assert "**Total Cost**" not in joined
+
+ def test_token_section_marks_partial_cost_as_floor(self):
+ """A priced row beside an unpriced one reports the total as a floor.
+
+ The failure this guards: summing only the rows that carry a cost and
+ presenting it as the run's bill. The unpriced rows are typically the
+ killed ones, which are also the ones that burned the most getting there.
+ """
+ task_results = [
+ {
+ "task_id": "priced",
+ "total_tokens": 1000,
+ "total_cost_usd": 0.25,
+ "agent_cost_usd": 0.25,
+ "cost_complete": True,
+ },
+ {
+ "task_id": "unpriced",
+ "total_tokens": 4_000_000,
+ "total_cost_usd": None,
+ "agent_cost_usd": None,
+ "cost_complete": False,
+ },
+ ]
+ joined = "\n".join(ReportGenerator._generate_token_usage_section(task_results))
+
+ assert "**Total Cost**: $0.2500 (floor" in joined
+ assert "1 task(s) have spend missing from this total" in joined
+
+ def test_token_section_breaks_out_eval_overhead(self):
+ """Judge spend is reported beside the agent bill, and folded into the total."""
+ task_results = [
+ {
+ "task_id": "task1",
+ "total_tokens": 1000,
+ "total_cost_usd": 1.25,
+ "agent_cost_usd": 1.0,
+ "cost_complete": True,
+ "judge_cost_usd": 0.25,
+ },
+ ]
+ joined = "\n".join(ReportGenerator._generate_token_usage_section(task_results))
+
+ assert "**Agent Cost**: $1.0000" in joined
+ assert "**Eval Overhead (judge + simulator)**: $0.2500" in joined
+ assert "**Total Cost**: $1.2500" in joined
+ # And the per-task column carries the whole bill, so it sums to that total.
+ assert "| task1 | 0 | 0 | 0 | 0 | 1,000 | $1.2500 |" in joined
+
def test_token_section_in_full_report(self):
"""Test that token section appears in generate_markdown when data is available."""
summary = RunSummary(