From 2de482f74e6ec468db11051cfb09220cf06b510c Mon Sep 17 00:00:00 2001 From: Ignazio De Santis Date: Sat, 25 Jul 2026 11:14:31 +0800 Subject: [PATCH 1/2] feat(llm_judge): multi-sample verdicts with median aggregation Add `samples` (1-9, default 1 = unchanged single-call behavior) to llm_judge. With N > 1 the judge grades the same rendered prompt N times; the criterion scores the median verdict, the sample closest to the median (earliest on ties) supplies rationale/findings/transcript, per-sample scores render in details, and token usage sums across samples. A sample that produces no verdict degrades to the median of the valid samples with a degraded note; when every sample fails, single-sample failure semantics apply unchanged (JudgeInfrastructureError still escalates, parse failures still score 0.0). agent_judge stays single-sample. --- docs/TASK_DEFINITION_GUIDE.md | 3 + src/coder_eval/criteria/llm_judge.py | 161 +++++++++++++++-- src/coder_eval/models/criteria.py | 21 +++ tests/test_llm_judge_criterion.py | 257 +++++++++++++++++++++++++++ 4 files changed, 428 insertions(+), 14 deletions(-) diff --git a/docs/TASK_DEFINITION_GUIDE.md b/docs/TASK_DEFINITION_GUIDE.md index c2de203a..60b7100c 100644 --- a/docs/TASK_DEFINITION_GUIDE.md +++ b/docs/TASK_DEFINITION_GUIDE.md @@ -834,8 +834,11 @@ Have an LLM grade the task against a rubric written in the task YAML. **Continuo | `model` | `anthropic.claude-sonnet-4-6` | Judge model id (vendor-prefixed; auto-translated per backend) | | `temperature` | `0.0` | Sampling temperature (0.0 = deterministic) | | `max_tokens` | `2000` | Maximum tokens in the judge's response | +| `samples` | `1` | Number of independent judge invocations (1–9). `1` = single call, unchanged behavior. With N > 1 the criterion scores the **median** of N sampled verdicts; rationale/findings come from the sample closest to the median (earliest on ties); per-sample scores land in `details`; token usage sums across samples. | | `max_file_chars` | `20000` | Per-file (and agent_output) truncation applied before building the prompt | +**Sampling the judge.** Even at `temperature: 0.0`, judge verdicts are not perfectly repeatable — the same artifacts can draw a strict or a lenient reading of the rubric on different calls. Set `samples: 3` (odd values recommended) to invoke the judge three times over the same rendered prompt and score the median verdict, damping a single outlier reading. Because every sample grades identical artifacts, the spread across samples is pure judge variance — unlike experiment-level `replicates`, which re-run the whole agent and mix agent variance into the same number. A sample that returns no verdict degrades to the median of the remaining valid samples (with a note in `details`) rather than scoring the criterion 0.0; only when every sample fails does the single-sample failure behavior apply. + **Transport selection.** The judge call is routed by the active `API_BACKEND`: | `API_BACKEND` | Credentials | Judge transport | diff --git a/src/coder_eval/criteria/llm_judge.py b/src/coder_eval/criteria/llm_judge.py index 73db9f4a..1270b375 100644 --- a/src/coder_eval/criteria/llm_judge.py +++ b/src/coder_eval/criteria/llm_judge.py @@ -1,9 +1,12 @@ """LLM-as-a-judge success criterion checker.""" import logging -from typing import TYPE_CHECKING +from collections.abc import Iterable +from statistics import median +from typing import TYPE_CHECKING, NamedTuple from coder_eval.criteria.base import BaseCriterion, CheckContext, register_criterion +from coder_eval.errors import JudgeInfrastructureError from coder_eval.evaluation.judge_anthropic import invoke_anthropic_judge from coder_eval.evaluation.judge_bedrock import invoke_bedrock_judge from coder_eval.evaluation.judge_context import ( @@ -113,6 +116,18 @@ def _check_impl( scrub_key = reference_code if criterion.include_reference else None + # Multi-sample grading (samples > 1) aggregates independent verdicts over + # the same rendered prompt; the default (1) stays on the single-call path + # below, unchanged. + if criterion.samples > 1: + return _grade_with_sampling( + criterion=criterion, + route=route, + user_msg=user_msg, + judge_ctx=judge_ctx, + scrub_key=scrub_key, + ) + # Attribute the judge's API call to ``JudgeCriterionResult.token_usage`` # from the usage the backend reported in its response. verdict, parse_error, raw_verdict_text, response_usage = _invoke_tool_channel( @@ -127,17 +142,6 @@ def _check_impl( # model could echo the reference back in an unparseable response, so we scrub it. scrubbed = scrub_reference(raw_verdict_text, scrub_key) - def _maybe_transcript() -> JudgeTranscript | None: - if not criterion.capture_transcript: - return None - return build_judge_transcript( - raw_verdict=raw_verdict_text, - max_chars=criterion.max_transcript_chars, - judge_system_prompt=_SYSTEM_PROMPT, - judge_prompt=user_msg, - scrub_key=scrub_key, - ) - if parse_error is not None: return JudgeCriterionResult( criterion_type=criterion.type, @@ -145,7 +149,7 @@ def _maybe_transcript() -> JudgeTranscript | None: score=0.0, details=scrubbed[:500], error=scrub_reference(parse_error, scrub_key), - transcript=_maybe_transcript(), + transcript=_build_transcript(criterion, raw_verdict_text, user_msg, scrub_key), token_usage=judge_usage, ) assert verdict is not None # parser contract: verdict is set iff parse_error is None @@ -157,7 +161,7 @@ def _maybe_transcript() -> JudgeTranscript | None: score=verdict.score, details=scrub_reference(details, scrub_key), findings=[scrub_reference(f, scrub_key) for f in verdict.findings], - transcript=_maybe_transcript(), + transcript=_build_transcript(criterion, raw_verdict_text, user_msg, scrub_key), token_usage=judge_usage, ) @@ -213,6 +217,135 @@ def _invoke_tool_channel( return None, err, f"(no verdict — {err})", response_usage +class _SampleOutcome(NamedTuple): + """One judge invocation's outcome, in ``_invoke_tool_channel`` return order.""" + + verdict: JudgeVerdict | None + parse_error: str | None + raw_verdict_text: str + response_usage: TokenUsage | None + + +def _grade_with_sampling( + *, + criterion: LLMJudgeCriterion, + route: "ApiRoute", + user_msg: str, + judge_ctx: JudgeContext, + scrub_key: str | None, +) -> JudgeCriterionResult: + """Invoke the judge ``criterion.samples`` times and score the median verdict. + + Every sample grades the SAME rendered prompt, so score spread across samples + is judge variance by construction — the median damps a single strict-or-lenient + outlier reading of the rubric. The representative sample (score closest to the + median, earliest on ties) supplies rationale/findings/transcript so the + persisted audit trail is a real verdict, never a synthetic blend. + + A sample that produces no verdict (a transport failure, or a response with no + usable ``submit_verdict`` call) degrades to the median of the remaining valid + samples with a note in ``details``. When NO sample produces a verdict, the + single-sample failure semantics apply: an infrastructure failure escalates + (``JudgeInfrastructureError`` propagates to ``FinalStatus.ERROR`` — judge infra + failure is not an agent failure), any other exception reaches + ``@handle_criterion_errors``, and all-parse-failures score 0.0 with the first + sample's diagnostic. + """ + outcomes: list[_SampleOutcome] = [] + infra_errors: list[JudgeInfrastructureError] = [] + unexpected_errors: list[Exception] = [] + for _ in range(criterion.samples): + try: + outcomes.append( + _SampleOutcome( + *_invoke_tool_channel( + criterion=criterion, + route=route, + system_msg=_SYSTEM_PROMPT, + user_msg=user_msg, + ) + ) + ) + except JudgeInfrastructureError as exc: + infra_errors.append(exc) + except Exception as exc: + unexpected_errors.append(exc) + + # Cost is real for every sample that returned a response, verdict or not. + token_usage = _sum_usage(o.response_usage for o in outcomes) + valid: list[tuple[JudgeVerdict, str]] = [(o.verdict, o.raw_verdict_text) for o in outcomes if o.verdict is not None] + + if not valid: + if infra_errors: + raise infra_errors[0] + if unexpected_errors: + raise unexpected_errors[0] + first = outcomes[0] + assert first.parse_error is not None # parser contract: verdict is set iff parse_error is None + scrubbed = scrub_reference(first.raw_verdict_text, scrub_key) + return JudgeCriterionResult( + criterion_type=criterion.type, + description=criterion.description, + score=0.0, + details=scrubbed[:500], + error=scrub_reference(first.parse_error, scrub_key), + transcript=_build_transcript(criterion, first.raw_verdict_text, user_msg, scrub_key), + token_usage=token_usage, + ) + + scores = [verdict.score for verdict, _ in valid] + median_score = float(median(scores)) + rep_verdict, rep_raw = min(valid, key=lambda pair: abs(pair[0].score - median_score)) + + degraded_notes = list(judge_ctx.degraded_notes) + failed_samples = criterion.samples - len(valid) + if failed_samples: + summary = f"median over {len(valid)} valid samples" + degraded_notes.append(f"{failed_samples}/{criterion.samples} judge samples produced no verdict; {summary}") + + details = format_details(median_score, rep_verdict.rationale, judge_ctx.missing_files, degraded_notes) + rendered_scores = ", ".join(f"{s:.3f}" for s in scores) + details = f"{details}\nsample_scores: [{rendered_scores}]" + return JudgeCriterionResult( + criterion_type=criterion.type, + description=criterion.description, + score=median_score, + details=scrub_reference(details, scrub_key), + findings=[scrub_reference(f, scrub_key) for f in rep_verdict.findings], + transcript=_build_transcript(criterion, rep_raw, user_msg, scrub_key), + token_usage=token_usage, + ) + + +def _sum_usage(usages: Iterable[TokenUsage | None]) -> TokenUsage | None: + """Field-wise sum of the reported usages; ``None`` when no sample reported any.""" + present = [u for u in usages if u is not None] + if not present: + return None + total = present[0] + for u in present[1:]: + total = total + u + return total + + +def _build_transcript( + criterion: LLMJudgeCriterion, + raw_verdict_text: str, + user_msg: str, + scrub_key: str | None, +) -> JudgeTranscript | None: + """Build the persisted judge transcript when ``capture_transcript`` is on.""" + if not criterion.capture_transcript: + return None + return build_judge_transcript( + raw_verdict=raw_verdict_text, + max_chars=criterion.max_transcript_chars, + judge_system_prompt=_SYSTEM_PROMPT, + judge_prompt=user_msg, + scrub_key=scrub_key, + ) + + def _render_user_message(prompt: str, context: JudgeContext) -> str: """Render the user-facing prompt envelope for the LLM judge.""" reference_block = "" diff --git a/src/coder_eval/models/criteria.py b/src/coder_eval/models/criteria.py index 2605e1cf..108fe46f 100644 --- a/src/coder_eval/models/criteria.py +++ b/src/coder_eval/models/criteria.py @@ -657,6 +657,10 @@ class LLMJudgeCriterion(BaseSuccessCriterion): Continuous scoring. LLM error or a "did not call submit_verdict" diagnostic -> 0.0 with error. Score is clamped to [0.0, 1.0] by the ``JudgeVerdict`` validator. Non-numeric / non-finite score -> 0.0 with error. + + Set ``samples`` > 1 to invoke the judge several times over the same rendered + prompt and score the median verdict (see the field description for the + aggregation semantics). """ # Strict YAML-key validation: catch typos at load time rather than silently @@ -760,6 +764,23 @@ class LLMJudgeCriterion(BaseSuccessCriterion): "(score + rationale + a handful of findings) without runaway." ), ) + samples: int = Field( + default=1, + ge=1, + le=9, + description=( + "Number of independent judge invocations. The default (1) keeps the single-call " + "behavior. With N > 1, the judge grades the same rendered prompt N times and the " + "criterion scores the MEDIAN of the sampled scores; rationale/findings/transcript " + "come from the sample whose score is closest to the median (earliest on ties), so " + "the audit trail is a real verdict rather than a synthetic blend. Per-sample scores " + "are rendered in ``details`` and token usage is summed across samples. With N >= 2, " + "a sample that produces no verdict degrades to the median of the valid samples " + "(with a note in ``details``); when every sample fails, single-sample failure " + "semantics apply unchanged. Odd N recommended — an even-N median averages the two " + "middle scores." + ), + ) max_file_chars: int = Field( default=20_000, gt=0, diff --git a/tests/test_llm_judge_criterion.py b/tests/test_llm_judge_criterion.py index a18b9318..f5db5189 100644 --- a/tests/test_llm_judge_criterion.py +++ b/tests/test_llm_judge_criterion.py @@ -1080,3 +1080,260 @@ def test_usage_extractors_degrade_non_numeric_values_without_raising() -> None: # A nested object is also non-coercible → degrades to 0 (here all-zero → None). assert token_usage_from_anthropic_dict({"usage": {"input_tokens": {}, "output_tokens": "x"}}) is None + + +# --- multi-sample judging (samples > 1) --- + + +def _sample_resp( + score: float, rationale: str = "ok", findings: list[str] | None = None, usage: dict | None = None +) -> dict: + """An Anthropic-shaped submit_verdict response for one judge sample.""" + verdict_input: dict[str, Any] = {"score": score, "rationale": rationale} + if findings is not None: + verdict_input["findings"] = findings + resp: dict[str, Any] = {"content": [{"type": "tool_use", "name": "submit_verdict", "input": verdict_input}]} + if usage is not None: + resp["usage"] = usage + return resp + + +def _no_verdict_resp(usage: dict | None = None) -> dict: + """A text-only response — the model returned prose instead of the tool call.""" + resp: dict[str, Any] = {"content": [{"type": "text", "text": "no verdict here"}]} + if usage is not None: + resp["usage"] = usage + return resp + + +def test_judge_samples_defaults_to_single_invocation(sandbox: Sandbox) -> None: + """Default samples=1: exactly one judge call, no aggregation artifacts in details.""" + criterion = LLMJudgeCriterion(description="x", prompt="grade") + assert criterion.samples == 1 + resp = _sample_resp(0.85, rationale="mostly correct") + with patch("coder_eval.criteria.llm_judge.invoke_anthropic_judge", return_value=resp) as m_anthropic: + result = SuccessChecker(sandbox, init_registry=False, route=DirectRoute()).check(criterion) + assert m_anthropic.call_count == 1 + assert result.score == 0.85 + assert result.error is None + assert "sample_scores" not in (result.details or "") + + +def test_judge_samples_bounds_validation() -> None: + from pydantic import ValidationError + + with pytest.raises(ValidationError): + LLMJudgeCriterion(description="x", prompt="grade", samples=0) + with pytest.raises(ValidationError): + LLMJudgeCriterion(description="x", prompt="grade", samples=10) + + +def test_judge_multi_sample_scores_median_odd(sandbox: Sandbox) -> None: + """samples=3: three invocations, median score wins, per-sample scores in details.""" + criterion = LLMJudgeCriterion(description="x", prompt="grade", samples=3) + responses = [_sample_resp(0.2, "low"), _sample_resp(0.9, "high"), _sample_resp(0.6, "mid")] + with patch("coder_eval.criteria.llm_judge.invoke_anthropic_judge", side_effect=responses) as m_anthropic: + result = SuccessChecker(sandbox, init_registry=False, route=DirectRoute()).check(criterion) + assert m_anthropic.call_count == 3 + assert result.score == 0.6 + assert result.error is None + details = result.details or "" + assert "rationale: mid" in details # representative = the median sample + assert "sample_scores: [0.200, 0.900, 0.600]" in details + # All samples succeeded — the degraded note must not appear. + assert "judge samples produced no verdict" not in details + + +def test_judge_multi_sample_reuses_identical_prompt(sandbox: Sandbox) -> None: + """Every sample grades the same rendered prompt with the same invocation params.""" + criterion = LLMJudgeCriterion(description="x", prompt="grade", samples=3) + responses = [_sample_resp(0.5), _sample_resp(0.5), _sample_resp(0.5)] + with patch("coder_eval.criteria.llm_judge.invoke_anthropic_judge", side_effect=responses) as m_anthropic: + SuccessChecker(sandbox, init_registry=False, route=DirectRoute()).check(criterion) + first_kwargs = m_anthropic.call_args_list[0].kwargs + for call in m_anthropic.call_args_list[1:]: + assert call.kwargs == first_kwargs + + +def test_judge_multi_sample_even_count_averages_middle(sandbox: Sandbox) -> None: + """Even N: median averages the two middle scores; representative is the closest sample.""" + criterion = LLMJudgeCriterion(description="x", prompt="grade", samples=4) + responses = [_sample_resp(0.2, "r1"), _sample_resp(0.4, "r2"), _sample_resp(0.6, "r3"), _sample_resp(0.8, "r4")] + with patch("coder_eval.criteria.llm_judge.invoke_anthropic_judge", side_effect=responses): + result = SuccessChecker(sandbox, init_registry=False, route=DirectRoute()).check(criterion) + assert result.score == pytest.approx(0.5) + # 0.4 and 0.6 are equidistant from the 0.5 median — the earlier sample wins. + assert "rationale: r2" in (result.details or "") + + +def test_judge_multi_sample_tie_break_is_first_sample(sandbox: Sandbox) -> None: + criterion = LLMJudgeCriterion(description="x", prompt="grade", samples=2) + responses = [_sample_resp(0.4, "first"), _sample_resp(0.6, "second")] + with patch("coder_eval.criteria.llm_judge.invoke_anthropic_judge", side_effect=responses): + result = SuccessChecker(sandbox, init_registry=False, route=DirectRoute()).check(criterion) + assert result.score == pytest.approx(0.5) + assert "rationale: first" in (result.details or "") + + +def test_judge_multi_sample_partial_no_verdict_degrades(sandbox: Sandbox) -> None: + """One sample returning no verdict must not zero the criterion — median of the valid ones.""" + criterion = LLMJudgeCriterion(description="x", prompt="grade", samples=3) + responses = [_sample_resp(0.8, "a"), _no_verdict_resp(), _sample_resp(0.6, "b")] + with patch("coder_eval.criteria.llm_judge.invoke_anthropic_judge", side_effect=responses): + result = SuccessChecker(sandbox, init_registry=False, route=DirectRoute()).check(criterion) + assert result.score == pytest.approx(0.7) + assert result.error is None + details = result.details or "" + assert "1/3 judge samples produced no verdict" in details + assert "median over 2 valid samples" in details + assert "sample_scores: [0.800, 0.600]" in details + + +def test_judge_multi_sample_two_samples_one_failure_degrades(sandbox: Sandbox) -> None: + """N=2 (the minimal multi-sample count) with one failure: the single valid score stands.""" + criterion = LLMJudgeCriterion(description="x", prompt="grade", samples=2) + responses = [_no_verdict_resp(), _sample_resp(0.8, "only valid")] + with patch("coder_eval.criteria.llm_judge.invoke_anthropic_judge", side_effect=responses): + result = SuccessChecker(sandbox, init_registry=False, route=DirectRoute()).check(criterion) + assert result.score == 0.8 + assert result.error is None + details = result.details or "" + assert "rationale: only valid" in details + assert "1/2 judge samples produced no verdict" in details + assert "median over 1 valid samples" in details + + +def test_judge_multi_sample_transport_exception_degrades(sandbox: Sandbox) -> None: + """One sample's transport failure must not void the others.""" + criterion = LLMJudgeCriterion(description="x", prompt="grade", samples=3) + responses = [_sample_resp(0.8), RuntimeError("gateway down"), _sample_resp(0.6)] + with patch("coder_eval.criteria.llm_judge.invoke_anthropic_judge", side_effect=responses): + result = SuccessChecker(sandbox, init_registry=False, route=DirectRoute()).check(criterion) + assert result.score == pytest.approx(0.7) + assert result.error is None + assert "1/3 judge samples produced no verdict" in (result.details or "") + + +def test_judge_multi_sample_infra_error_degrades_when_verdicts_exist(sandbox: Sandbox) -> None: + """An infra failure on one sample degrades (never scores 0.0) when other samples produced verdicts.""" + from coder_eval.errors import JudgeInfrastructureError + + criterion = LLMJudgeCriterion(description="x", prompt="grade", samples=3) + responses = [JudgeInfrastructureError("api error"), _sample_resp(0.5), _sample_resp(0.9)] + with patch("coder_eval.criteria.llm_judge.invoke_anthropic_judge", side_effect=responses): + result = SuccessChecker(sandbox, init_registry=False, route=DirectRoute()).check(criterion) + assert result.score == pytest.approx(0.7) + assert result.error is None + + +def test_judge_multi_sample_all_no_verdict_keeps_failure_path(sandbox: Sandbox) -> None: + """Every sample failing to call submit_verdict keeps the single-sample failure semantics.""" + criterion = LLMJudgeCriterion(description="x", prompt="grade", samples=3) + responses = [_no_verdict_resp(), _no_verdict_resp(), _no_verdict_resp()] + with patch("coder_eval.criteria.llm_judge.invoke_anthropic_judge", side_effect=responses): + result = SuccessChecker(sandbox, init_registry=False, route=DirectRoute()).check(criterion) + assert result.score == 0.0 + assert result.error == "Judge did not call submit_verdict" + + +def test_judge_multi_sample_all_fail_reports_first_sample_diagnostic(sandbox: Sandbox) -> None: + """When every sample fails to produce a verdict, the FIRST sample's diagnostic is reported.""" + criterion = LLMJudgeCriterion(description="x", prompt="grade", samples=2) + bad_args = {"content": [{"type": "tool_use", "name": "submit_verdict", "input": {"score": "not numeric"}}]} + responses = [bad_args, _no_verdict_resp()] + with patch("coder_eval.criteria.llm_judge.invoke_anthropic_judge", side_effect=responses): + result = SuccessChecker(sandbox, init_registry=False, route=DirectRoute()).check(criterion) + assert result.score == 0.0 + assert result.error is not None + assert "score field is not a number" in result.error # sample #1's diagnostic, not sample #2's + + +def test_judge_multi_sample_all_infra_errors_escalate(sandbox: Sandbox) -> None: + """All samples failing on infrastructure escalates — judge infra failure is not an agent failure.""" + from coder_eval.errors import JudgeInfrastructureError + + criterion = LLMJudgeCriterion(description="x", prompt="grade", samples=2) + responses = [JudgeInfrastructureError("api error"), JudgeInfrastructureError("api error")] + with ( + patch("coder_eval.criteria.llm_judge.invoke_anthropic_judge", side_effect=responses), + pytest.raises(JudgeInfrastructureError), + ): + SuccessChecker(sandbox, init_registry=False, route=DirectRoute()).check(criterion) + + +def test_judge_multi_sample_mixed_all_fail_prefers_infra_escalation(sandbox: Sandbox) -> None: + """No verdict anywhere + an infra failure in the mix: escalate rather than score 0.0.""" + from coder_eval.errors import JudgeInfrastructureError + + criterion = LLMJudgeCriterion(description="x", prompt="grade", samples=2) + responses = [_no_verdict_resp(), JudgeInfrastructureError("api error")] + with ( + patch("coder_eval.criteria.llm_judge.invoke_anthropic_judge", side_effect=responses), + pytest.raises(JudgeInfrastructureError), + ): + SuccessChecker(sandbox, init_registry=False, route=DirectRoute()).check(criterion) + + +def test_judge_multi_sample_all_unexpected_errors_map_to_score_zero(sandbox: Sandbox) -> None: + """Non-infra exceptions on every sample reach @handle_criterion_errors, as with samples=1.""" + criterion = LLMJudgeCriterion(description="x", prompt="grade", samples=2) + responses = [RuntimeError("gateway down"), RuntimeError("gateway down")] + with patch("coder_eval.criteria.llm_judge.invoke_anthropic_judge", side_effect=responses): + result = SuccessChecker(sandbox, init_registry=False, route=DirectRoute()).check(criterion) + assert result.score == 0.0 + assert result.error is not None + assert "gateway down" in result.error + + +def test_judge_multi_sample_token_usage_summed(sandbox: Sandbox) -> None: + """Usage sums across every sample that returned a response — including no-verdict samples.""" + from coder_eval.models import JudgeCriterionResult + + criterion = LLMJudgeCriterion(description="x", prompt="grade", samples=3) + usage = {"input_tokens": 100, "output_tokens": 10} + responses = [_sample_resp(0.8, usage=usage), _no_verdict_resp(usage=usage), _sample_resp(0.6, usage=usage)] + with patch("coder_eval.criteria.llm_judge.invoke_anthropic_judge", side_effect=responses): + result = SuccessChecker(sandbox, init_registry=False, route=DirectRoute()).check(criterion) + assert isinstance(result, JudgeCriterionResult) + assert result.token_usage is not None + assert result.token_usage.uncached_input_tokens == 300 + assert result.token_usage.output_tokens == 30 + + +def test_judge_multi_sample_representative_supplies_findings_and_transcript(sandbox: Sandbox) -> None: + """Findings and the persisted transcript come from the representative sample, not a blend.""" + criterion = LLMJudgeCriterion(description="x", prompt="grade", samples=3) + responses = [ + _sample_resp(0.2, "low", findings=["low finding"]), + _sample_resp(0.6, "mid", findings=["mid finding"]), + _sample_resp(0.9, "high", findings=["high finding"]), + ] + with patch("coder_eval.criteria.llm_judge.invoke_anthropic_judge", side_effect=responses): + result = SuccessChecker(sandbox, init_registry=False, route=DirectRoute()).check(criterion) + assert result.score == 0.6 + assert getattr(result, "findings", []) == ["mid finding"] + transcript = getattr(result, "transcript", None) + assert transcript is not None + assert '"score":0.6' in transcript.raw_verdict + + +def test_judge_multi_sample_scrubs_reference_everywhere(sandbox: Sandbox) -> None: + """Reference scrubbing covers details, findings, and the transcript with samples > 1.""" + sentinel = "REF_LEAK_MULTI_SAMPLE_321" + criterion = LLMJudgeCriterion(description="x", prompt="grade", include_reference=True, samples=3) + responses = [ + _sample_resp(0.4, f"echoed {sentinel}"), + _sample_resp(0.5, f"echoed {sentinel}", findings=[f"finding with {sentinel}"]), + _sample_resp(0.6, f"echoed {sentinel}"), + ] + with patch("coder_eval.criteria.llm_judge.invoke_anthropic_judge", side_effect=responses): + result = SuccessChecker(sandbox, init_registry=False, route=DirectRoute()).check( + criterion, reference_code=sentinel + ) + assert result.score == 0.5 + assert sentinel not in (result.details or "") + for finding in getattr(result, "findings", []) or []: + assert sentinel not in finding + transcript = getattr(result, "transcript", None) + assert transcript is not None + assert sentinel not in transcript.raw_verdict From cc3988792c36bf692065a71b733e3535976c7376 Mon Sep 17 00:00:00 2001 From: Ignazio De Santis Date: Wed, 29 Jul 2026 10:24:33 +0800 Subject: [PATCH 2/2] fix(llm_judge): concurrent samples, single judge path, loud sample failures Review follow-up (PR #53): - Dispatch judge samples concurrently (ThreadPoolExecutor); futures are collected in submission order so tie-breaks, the first-sample diagnostic, and error triage stay deterministic. Wall-clock stays ~ one judge call, so samples no longer eats into task_timeout. Pinned by a sleeping-fake wall-clock regression test. - Collapse the samples>1 dispatch into the one grading path: N=1 is the one-element case (median of one, summed usage of one); sample_scores and the degraded note are gated on samples > 1. Deletes both duplicated JudgeCriterionResult construction tails; single-sample tests unmodified. - Stop discarding sample-failure diagnostics: every swallowed exception is logged at WARNING with exc_info and named (type + message) in the degraded note; all-fail triage keeps first-seen errors directly. - _invoke_tool_channel returns _SampleOutcome (keyword-constructed, no positional splat); _build_transcript is keyword-only; drop the redundant float() around median() and the class-docstring aggregation restatement. - Tests: Bedrock-arm multi-sample (escalate-vs-degrade), cache-bucket usage summing, token_usage-None, all-fail usage+transcript audit, concurrency wall-clock; ordering-pinning tests patch a serial executor because concurrent dispatch makes mock side_effect response-to-sample mapping scheduler-dependent. - Docs: guide documents concurrency + the Nx judge spend that max_usd does not gate, and why samples is llm_judge-only; field description gains the le=9 rationale and the sample_per_stratum disambiguation. --- docs/TASK_DEFINITION_GUIDE.md | 4 +- src/coder_eval/criteria/llm_judge.py | 212 ++++++++++++++------------- src/coder_eval/models/criteria.py | 11 +- tests/test_llm_judge_criterion.py | 192 ++++++++++++++++++++++-- 4 files changed, 301 insertions(+), 118 deletions(-) diff --git a/docs/TASK_DEFINITION_GUIDE.md b/docs/TASK_DEFINITION_GUIDE.md index 60b7100c..6e97cc17 100644 --- a/docs/TASK_DEFINITION_GUIDE.md +++ b/docs/TASK_DEFINITION_GUIDE.md @@ -834,11 +834,13 @@ Have an LLM grade the task against a rubric written in the task YAML. **Continuo | `model` | `anthropic.claude-sonnet-4-6` | Judge model id (vendor-prefixed; auto-translated per backend) | | `temperature` | `0.0` | Sampling temperature (0.0 = deterministic) | | `max_tokens` | `2000` | Maximum tokens in the judge's response | -| `samples` | `1` | Number of independent judge invocations (1–9). `1` = single call, unchanged behavior. With N > 1 the criterion scores the **median** of N sampled verdicts; rationale/findings come from the sample closest to the median (earliest on ties); per-sample scores land in `details`; token usage sums across samples. | +| `samples` | `1` | Number of independent judge invocations (1–9). `1` = single call, unchanged behavior. With N > 1 the criterion scores the **median** of N sampled verdicts; rationale/findings come from the sample closest to the median (earliest on ties); per-sample scores land in `details`; token usage sums across samples. Samples run concurrently (wall-clock ≈ one call); judge spend multiplies by N and is not gated by `max_usd`. | | `max_file_chars` | `20000` | Per-file (and agent_output) truncation applied before building the prompt | **Sampling the judge.** Even at `temperature: 0.0`, judge verdicts are not perfectly repeatable — the same artifacts can draw a strict or a lenient reading of the rubric on different calls. Set `samples: 3` (odd values recommended) to invoke the judge three times over the same rendered prompt and score the median verdict, damping a single outlier reading. Because every sample grades identical artifacts, the spread across samples is pure judge variance — unlike experiment-level `replicates`, which re-run the whole agent and mix agent variance into the same number. A sample that returns no verdict degrades to the median of the remaining valid samples (with a note in `details`) rather than scoring the criterion 0.0; only when every sample fails does the single-sample failure behavior apply. +**Cost and latency.** The N samples are dispatched concurrently, so wall-clock stays close to a single judge call regardless of N — `samples` does not eat into `task_timeout` beyond the one-call baseline. Judge **spend** still multiplies by N, and judge usage is deliberately excluded from `run_limits.max_usd` (that cap meters the *agent's* bill): on a dataset-fanned suite, `samples: 9` multiplies ungated judge cost 9× per row — size N with that in mind. `samples` is deliberately `llm_judge`-only: an `agent_judge` verdict costs a full tool-using sub-agent run, so multi-sampling it would multiply a far larger bill for the same variance signal — `agent_judge` does not accept the key (strict YAML validation rejects it at load time). + **Transport selection.** The judge call is routed by the active `API_BACKEND`: | `API_BACKEND` | Credentials | Judge transport | diff --git a/src/coder_eval/criteria/llm_judge.py b/src/coder_eval/criteria/llm_judge.py index 1270b375..59370219 100644 --- a/src/coder_eval/criteria/llm_judge.py +++ b/src/coder_eval/criteria/llm_judge.py @@ -2,6 +2,7 @@ import logging from collections.abc import Iterable +from concurrent.futures import ThreadPoolExecutor from statistics import median from typing import TYPE_CHECKING, NamedTuple @@ -116,54 +117,31 @@ def _check_impl( scrub_key = reference_code if criterion.include_reference else None - # Multi-sample grading (samples > 1) aggregates independent verdicts over - # the same rendered prompt; the default (1) stays on the single-call path - # below, unchanged. - if criterion.samples > 1: - return _grade_with_sampling( - criterion=criterion, - route=route, - user_msg=user_msg, - judge_ctx=judge_ctx, - scrub_key=scrub_key, - ) - - # Attribute the judge's API call to ``JudgeCriterionResult.token_usage`` - # from the usage the backend reported in its response. - verdict, parse_error, raw_verdict_text, response_usage = _invoke_tool_channel( + # One grading path for every N: samples=1 is the one-element case whose + # median is its own score and whose summed usage is its own usage. + return _grade_with_sampling( criterion=criterion, route=route, - system_msg=_SYSTEM_PROMPT, user_msg=user_msg, + judge_ctx=judge_ctx, + scrub_key=scrub_key, ) - judge_usage = response_usage - # Sanitize any raw model text we persist to CriterionResult.details. A misbehaving - # model could echo the reference back in an unparseable response, so we scrub it. - scrubbed = scrub_reference(raw_verdict_text, scrub_key) - if parse_error is not None: - return JudgeCriterionResult( - criterion_type=criterion.type, - description=criterion.description, - score=0.0, - details=scrubbed[:500], - error=scrub_reference(parse_error, scrub_key), - transcript=_build_transcript(criterion, raw_verdict_text, user_msg, scrub_key), - token_usage=judge_usage, - ) - assert verdict is not None # parser contract: verdict is set iff parse_error is None +class _SampleOutcome(NamedTuple): + """One judge invocation's outcome. - details = format_details(verdict.score, verdict.rationale, judge_ctx.missing_files, judge_ctx.degraded_notes) - return JudgeCriterionResult( - criterion_type=criterion.type, - description=criterion.description, - score=verdict.score, - details=scrub_reference(details, scrub_key), - findings=[scrub_reference(f, scrub_key) for f in verdict.findings], - transcript=_build_transcript(criterion, raw_verdict_text, user_msg, scrub_key), - token_usage=judge_usage, - ) + ``raw_verdict_text`` is the JSON-dumped verdict for the transcript when a + verdict was parsed, or a fallback marker when the model failed to call the + tool — preserves the "judge transcript carries the structured payload" + invariant. ``response_usage`` is the usage the model reported (``None`` + when the backend surfaced none). + """ + + verdict: JudgeVerdict | None + parse_error: str | None + raw_verdict_text: str + response_usage: TokenUsage | None def _invoke_tool_channel( @@ -172,16 +150,8 @@ def _invoke_tool_channel( route: "ApiRoute | None", system_msg: str, user_msg: str, -) -> tuple[JudgeVerdict | None, str | None, str, TokenUsage | None]: - """Dispatch the tool-channel invocation by route. - - Returns ``(verdict, parse_error, raw_verdict_text, response_usage)``. - ``raw_verdict_text`` is the JSON-dumped verdict for the transcript when - present, or a fallback marker when the model failed to call the tool — - preserves the "judge transcript carries the structured payload" invariant. - ``response_usage`` is the usage the model reported (``None`` when the - backend surfaced none). - """ +) -> _SampleOutcome: + """Dispatch the tool-channel invocation by route.""" response_usage: TokenUsage | None match route: case BedrockRoute(): @@ -210,20 +180,26 @@ def _invoke_tool_channel( case _: # route is None or an unexpected type — the unconfigured-arm guard in # _check_impl handles None before dispatch, so this is defensive only. - return None, "llm_judge: no usable API route", "(no route)", None + return _SampleOutcome( + verdict=None, + parse_error="llm_judge: no usable API route", + raw_verdict_text="(no route)", + response_usage=None, + ) if verdict is not None: - return verdict, None, verdict.model_dump_json(), response_usage - return None, err, f"(no verdict — {err})", response_usage - - -class _SampleOutcome(NamedTuple): - """One judge invocation's outcome, in ``_invoke_tool_channel`` return order.""" - - verdict: JudgeVerdict | None - parse_error: str | None - raw_verdict_text: str - response_usage: TokenUsage | None + return _SampleOutcome( + verdict=verdict, + parse_error=None, + raw_verdict_text=verdict.model_dump_json(), + response_usage=response_usage, + ) + return _SampleOutcome( + verdict=None, + parse_error=err, + raw_verdict_text=f"(no verdict — {err})", + response_usage=response_usage, + ) def _grade_with_sampling( @@ -236,50 +212,63 @@ def _grade_with_sampling( ) -> JudgeCriterionResult: """Invoke the judge ``criterion.samples`` times and score the median verdict. - Every sample grades the SAME rendered prompt, so score spread across samples - is judge variance by construction — the median damps a single strict-or-lenient - outlier reading of the rubric. The representative sample (score closest to the - median, earliest on ties) supplies rationale/findings/transcript so the - persisted audit trail is a real verdict, never a synthetic blend. + The single grading path: ``samples: 1`` is the one-element case (its median + is its own score, the summed usage is its own usage), so single-sample and + multi-sample results are built by the same code. Samples are independent and + grade the SAME rendered prompt, so they are dispatched concurrently — score + spread across samples is judge variance by construction, and wall-clock stays + close to one judge call. Futures are collected in SUBMISSION order, not + completion order, so the representative tie-break (earliest sample), the + first-sample diagnostic, and error triage stay deterministic. A sample that produces no verdict (a transport failure, or a response with no usable ``submit_verdict`` call) degrades to the median of the remaining valid - samples with a note in ``details``. When NO sample produces a verdict, the - single-sample failure semantics apply: an infrastructure failure escalates - (``JudgeInfrastructureError`` propagates to ``FinalStatus.ERROR`` — judge infra - failure is not an agent failure), any other exception reaches - ``@handle_criterion_errors``, and all-parse-failures score 0.0 with the first - sample's diagnostic. + samples — every swallowed exception is logged at WARNING and named in the + degraded note so a systematic first-party bug stays loud. When NO sample + produces a verdict, the single-sample failure semantics apply: an + infrastructure failure escalates (``JudgeInfrastructureError`` propagates to + ``FinalStatus.ERROR`` — judge infra failure is not an agent failure), any + other exception reaches ``@handle_criterion_errors``, and all-parse-failures + score 0.0 with the first sample's diagnostic. """ outcomes: list[_SampleOutcome] = [] - infra_errors: list[JudgeInfrastructureError] = [] - unexpected_errors: list[Exception] = [] - for _ in range(criterion.samples): - try: - outcomes.append( - _SampleOutcome( - *_invoke_tool_channel( - criterion=criterion, - route=route, - system_msg=_SYSTEM_PROMPT, - user_msg=user_msg, - ) - ) + failure_labels: list[str] = [] + first_infra: JudgeInfrastructureError | None = None + first_unexpected: Exception | None = None + with ThreadPoolExecutor(max_workers=criterion.samples) as pool: + futures = [ + pool.submit( + _invoke_tool_channel, + criterion=criterion, + route=route, + system_msg=_SYSTEM_PROMPT, + user_msg=user_msg, ) - except JudgeInfrastructureError as exc: - infra_errors.append(exc) - except Exception as exc: - unexpected_errors.append(exc) + for _ in range(criterion.samples) + ] + for future in futures: + try: + outcomes.append(future.result()) + except JudgeInfrastructureError as exc: + logger.warning("llm_judge sample failed: %s", exc, exc_info=True) + failure_labels.append(f"{type(exc).__name__}: {exc}") + if first_infra is None: + first_infra = exc + except Exception as exc: + logger.warning("llm_judge sample failed: %s", exc, exc_info=True) + failure_labels.append(f"{type(exc).__name__}: {exc}") + if first_unexpected is None: + first_unexpected = exc # Cost is real for every sample that returned a response, verdict or not. token_usage = _sum_usage(o.response_usage for o in outcomes) valid: list[tuple[JudgeVerdict, str]] = [(o.verdict, o.raw_verdict_text) for o in outcomes if o.verdict is not None] if not valid: - if infra_errors: - raise infra_errors[0] - if unexpected_errors: - raise unexpected_errors[0] + if first_infra is not None: + raise first_infra + if first_unexpected is not None: + raise first_unexpected first = outcomes[0] assert first.parse_error is not None # parser contract: verdict is set iff parse_error is None scrubbed = scrub_reference(first.raw_verdict_text, scrub_key) @@ -289,30 +278,46 @@ def _grade_with_sampling( score=0.0, details=scrubbed[:500], error=scrub_reference(first.parse_error, scrub_key), - transcript=_build_transcript(criterion, first.raw_verdict_text, user_msg, scrub_key), + transcript=_build_transcript( + criterion=criterion, + raw_verdict_text=first.raw_verdict_text, + user_msg=user_msg, + scrub_key=scrub_key, + ), token_usage=token_usage, ) scores = [verdict.score for verdict, _ in valid] - median_score = float(median(scores)) + median_score = median(scores) rep_verdict, rep_raw = min(valid, key=lambda pair: abs(pair[0].score - median_score)) degraded_notes = list(judge_ctx.degraded_notes) failed_samples = criterion.samples - len(valid) - if failed_samples: - summary = f"median over {len(valid)} valid samples" - degraded_notes.append(f"{failed_samples}/{criterion.samples} judge samples produced no verdict; {summary}") + if criterion.samples > 1 and failed_samples: + note = ( + f"{failed_samples}/{criterion.samples} judge samples produced no verdict; " + f"median over {len(valid)} valid samples" + ) + if failure_labels: + note = f"{note} ({'; '.join(failure_labels)})" + degraded_notes.append(note) details = format_details(median_score, rep_verdict.rationale, judge_ctx.missing_files, degraded_notes) - rendered_scores = ", ".join(f"{s:.3f}" for s in scores) - details = f"{details}\nsample_scores: [{rendered_scores}]" + if criterion.samples > 1: + rendered_scores = ", ".join(f"{s:.3f}" for s in scores) + details = f"{details}\nsample_scores: [{rendered_scores}]" return JudgeCriterionResult( criterion_type=criterion.type, description=criterion.description, score=median_score, details=scrub_reference(details, scrub_key), findings=[scrub_reference(f, scrub_key) for f in rep_verdict.findings], - transcript=_build_transcript(criterion, rep_raw, user_msg, scrub_key), + transcript=_build_transcript( + criterion=criterion, + raw_verdict_text=rep_raw, + user_msg=user_msg, + scrub_key=scrub_key, + ), token_usage=token_usage, ) @@ -329,6 +334,7 @@ def _sum_usage(usages: Iterable[TokenUsage | None]) -> TokenUsage | None: def _build_transcript( + *, criterion: LLMJudgeCriterion, raw_verdict_text: str, user_msg: str, diff --git a/src/coder_eval/models/criteria.py b/src/coder_eval/models/criteria.py index 108fe46f..097bfb5d 100644 --- a/src/coder_eval/models/criteria.py +++ b/src/coder_eval/models/criteria.py @@ -657,10 +657,6 @@ class LLMJudgeCriterion(BaseSuccessCriterion): Continuous scoring. LLM error or a "did not call submit_verdict" diagnostic -> 0.0 with error. Score is clamped to [0.0, 1.0] by the ``JudgeVerdict`` validator. Non-numeric / non-finite score -> 0.0 with error. - - Set ``samples`` > 1 to invoke the judge several times over the same rendered - prompt and score the median verdict (see the field description for the - aggregation semantics). """ # Strict YAML-key validation: catch typos at load time rather than silently @@ -778,7 +774,12 @@ class LLMJudgeCriterion(BaseSuccessCriterion): "a sample that produces no verdict degrades to the median of the valid samples " "(with a note in ``details``); when every sample fails, single-sample failure " "semantics apply unchanged. Odd N recommended — an even-N median averages the two " - "middle scores." + "middle scores. Samples run concurrently, so wall-clock stays close to one judge " + "call; judge SPEND still multiplies by N and is not gated by ``run_limits.max_usd``. " + "Capped at 9 because the median's variance damping flattens well before that while " + "cost grows linearly — past 9 you pay for noise reduction you can no longer measure. " + "Unrelated to the dataset-level ``sample_per_stratum`` / CLI ``--sample`` (those " + "subsample dataset ROWS; this re-samples the judge's verdict on one row)." ), ) max_file_chars: int = Field( diff --git a/tests/test_llm_judge_criterion.py b/tests/test_llm_judge_criterion.py index f5db5189..d531c475 100644 --- a/tests/test_llm_judge_criterion.py +++ b/tests/test_llm_judge_criterion.py @@ -3,6 +3,7 @@ from __future__ import annotations import json +from concurrent.futures import Future from datetime import datetime from pathlib import Path from typing import Any @@ -1106,6 +1107,39 @@ def _no_verdict_resp(usage: dict | None = None) -> dict: return resp +class _SerialExecutor: + """Inline-executing stand-in for ``ThreadPoolExecutor``, with real Futures. + + Concurrent dispatch makes a mock ``side_effect`` list's response-to-sample + mapping scheduler-dependent (whichever worker thread calls first consumes + the next list item). Ordering-pinning tests — sample_scores order, + earliest-sample tie-breaks, the first-sample diagnostic — patch this + executor so response N deterministically lands on sample N. Everything the + production code touches (context manager, ``submit`` returning a ``Future`` + that carries a result or an exception) behaves identically. + """ + + def __init__(self, max_workers: int | None = None) -> None: + self.max_workers = max_workers + + def __enter__(self) -> _SerialExecutor: + return self + + def __exit__(self, *exc_info: object) -> None: + return None + + def submit(self, fn: Any, /, *args: Any, **kwargs: Any) -> Future[Any]: + future: Future[Any] = Future() + try: + future.set_result(fn(*args, **kwargs)) + except Exception as exc: + future.set_exception(exc) + return future + + +_SERIAL_EXECUTOR_PATCH = "coder_eval.criteria.llm_judge.ThreadPoolExecutor" + + def test_judge_samples_defaults_to_single_invocation(sandbox: Sandbox) -> None: """Default samples=1: exactly one judge call, no aggregation artifacts in details.""" criterion = LLMJudgeCriterion(description="x", prompt="grade") @@ -1132,7 +1166,10 @@ def test_judge_multi_sample_scores_median_odd(sandbox: Sandbox) -> None: """samples=3: three invocations, median score wins, per-sample scores in details.""" criterion = LLMJudgeCriterion(description="x", prompt="grade", samples=3) responses = [_sample_resp(0.2, "low"), _sample_resp(0.9, "high"), _sample_resp(0.6, "mid")] - with patch("coder_eval.criteria.llm_judge.invoke_anthropic_judge", side_effect=responses) as m_anthropic: + with ( + patch(_SERIAL_EXECUTOR_PATCH, _SerialExecutor), + patch("coder_eval.criteria.llm_judge.invoke_anthropic_judge", side_effect=responses) as m_anthropic, + ): result = SuccessChecker(sandbox, init_registry=False, route=DirectRoute()).check(criterion) assert m_anthropic.call_count == 3 assert result.score == 0.6 @@ -1159,7 +1196,10 @@ def test_judge_multi_sample_even_count_averages_middle(sandbox: Sandbox) -> None """Even N: median averages the two middle scores; representative is the closest sample.""" criterion = LLMJudgeCriterion(description="x", prompt="grade", samples=4) responses = [_sample_resp(0.2, "r1"), _sample_resp(0.4, "r2"), _sample_resp(0.6, "r3"), _sample_resp(0.8, "r4")] - with patch("coder_eval.criteria.llm_judge.invoke_anthropic_judge", side_effect=responses): + with ( + patch(_SERIAL_EXECUTOR_PATCH, _SerialExecutor), + patch("coder_eval.criteria.llm_judge.invoke_anthropic_judge", side_effect=responses), + ): result = SuccessChecker(sandbox, init_registry=False, route=DirectRoute()).check(criterion) assert result.score == pytest.approx(0.5) # 0.4 and 0.6 are equidistant from the 0.5 median — the earlier sample wins. @@ -1169,7 +1209,10 @@ def test_judge_multi_sample_even_count_averages_middle(sandbox: Sandbox) -> None def test_judge_multi_sample_tie_break_is_first_sample(sandbox: Sandbox) -> None: criterion = LLMJudgeCriterion(description="x", prompt="grade", samples=2) responses = [_sample_resp(0.4, "first"), _sample_resp(0.6, "second")] - with patch("coder_eval.criteria.llm_judge.invoke_anthropic_judge", side_effect=responses): + with ( + patch(_SERIAL_EXECUTOR_PATCH, _SerialExecutor), + patch("coder_eval.criteria.llm_judge.invoke_anthropic_judge", side_effect=responses), + ): result = SuccessChecker(sandbox, init_registry=False, route=DirectRoute()).check(criterion) assert result.score == pytest.approx(0.5) assert "rationale: first" in (result.details or "") @@ -1179,7 +1222,10 @@ def test_judge_multi_sample_partial_no_verdict_degrades(sandbox: Sandbox) -> Non """One sample returning no verdict must not zero the criterion — median of the valid ones.""" criterion = LLMJudgeCriterion(description="x", prompt="grade", samples=3) responses = [_sample_resp(0.8, "a"), _no_verdict_resp(), _sample_resp(0.6, "b")] - with patch("coder_eval.criteria.llm_judge.invoke_anthropic_judge", side_effect=responses): + with ( + patch(_SERIAL_EXECUTOR_PATCH, _SerialExecutor), + patch("coder_eval.criteria.llm_judge.invoke_anthropic_judge", side_effect=responses), + ): result = SuccessChecker(sandbox, init_registry=False, route=DirectRoute()).check(criterion) assert result.score == pytest.approx(0.7) assert result.error is None @@ -1203,15 +1249,32 @@ def test_judge_multi_sample_two_samples_one_failure_degrades(sandbox: Sandbox) - assert "median over 1 valid samples" in details -def test_judge_multi_sample_transport_exception_degrades(sandbox: Sandbox) -> None: - """One sample's transport failure must not void the others.""" +def test_judge_multi_sample_transport_exception_degrades(sandbox: Sandbox, caplog: pytest.LogCaptureFixture) -> None: + """One sample's transport failure must not void the others — but it must stay loud. + + The swallowed exception is logged at WARNING with exc_info and its type name + lands in the degraded note, so a systematic first-party bug that trips on a + fraction of responses can never ship as a silently-shifted green median. + """ + import logging + criterion = LLMJudgeCriterion(description="x", prompt="grade", samples=3) responses = [_sample_resp(0.8), RuntimeError("gateway down"), _sample_resp(0.6)] - with patch("coder_eval.criteria.llm_judge.invoke_anthropic_judge", side_effect=responses): + with ( + caplog.at_level(logging.WARNING, logger="coder_eval.criteria.llm_judge"), + patch("coder_eval.criteria.llm_judge.invoke_anthropic_judge", side_effect=responses), + ): result = SuccessChecker(sandbox, init_registry=False, route=DirectRoute()).check(criterion) assert result.score == pytest.approx(0.7) assert result.error is None - assert "1/3 judge samples produced no verdict" in (result.details or "") + details = result.details or "" + assert "1/3 judge samples produced no verdict" in details + assert "RuntimeError: gateway down" in details # the failure is NAMED, not just counted + warning_records = [ + r for r in caplog.records if r.levelno == logging.WARNING and "llm_judge sample failed" in r.getMessage() + ] + assert len(warning_records) == 1 + assert warning_records[0].exc_info is not None # traceback preserved for triage def test_judge_multi_sample_infra_error_degrades_when_verdicts_exist(sandbox: Sandbox) -> None: @@ -1241,7 +1304,10 @@ def test_judge_multi_sample_all_fail_reports_first_sample_diagnostic(sandbox: Sa criterion = LLMJudgeCriterion(description="x", prompt="grade", samples=2) bad_args = {"content": [{"type": "tool_use", "name": "submit_verdict", "input": {"score": "not numeric"}}]} responses = [bad_args, _no_verdict_resp()] - with patch("coder_eval.criteria.llm_judge.invoke_anthropic_judge", side_effect=responses): + with ( + patch(_SERIAL_EXECUTOR_PATCH, _SerialExecutor), + patch("coder_eval.criteria.llm_judge.invoke_anthropic_judge", side_effect=responses), + ): result = SuccessChecker(sandbox, init_registry=False, route=DirectRoute()).check(criterion) assert result.score == 0.0 assert result.error is not None @@ -1337,3 +1403,111 @@ def test_judge_multi_sample_scrubs_reference_everywhere(sandbox: Sandbox) -> Non transcript = getattr(result, "transcript", None) assert transcript is not None assert sentinel not in transcript.raw_verdict + + +def test_judge_multi_sample_bedrock_route(sandbox: Sandbox) -> None: + """samples>1 on the Bedrock arm — the nightly's route, and the invoker whose + every failure mode is a ``JudgeInfrastructureError`` (the escalate-vs-degrade branch).""" + from coder_eval.errors import JudgeInfrastructureError + from coder_eval.models.routing import BedrockRoute + + criterion = LLMJudgeCriterion(description="x", prompt="grade", samples=3) + responses = [_sample_resp(0.2, "low"), JudgeInfrastructureError("bedrock api down"), _sample_resp(0.6, "mid")] + route = BedrockRoute(bearer_token="t", region="us-east-1") + with ( + patch("coder_eval.criteria.llm_judge.invoke_bedrock_judge", side_effect=responses) as m_bedrock, + patch("coder_eval.criteria.llm_judge.invoke_anthropic_judge") as m_anthropic, + ): + result = SuccessChecker(sandbox, init_registry=False, route=route).check(criterion) + assert m_bedrock.call_count == 3 + assert m_anthropic.call_count == 0 + # One infra failure with valid verdicts remaining degrades — never escalates. + assert result.score == pytest.approx(0.4) + assert result.error is None + details = result.details or "" + assert "1/3 judge samples produced no verdict" in details + assert "JudgeInfrastructureError: bedrock api down" in details + assert "sample_scores" in details + + +def test_judge_multi_sample_token_usage_sums_cache_buckets(sandbox: Sandbox) -> None: + """Cache-token buckets sum across samples — N identical prompts are exactly + what populates cache_read on every sample after the first.""" + from coder_eval.models import JudgeCriterionResult + + criterion = LLMJudgeCriterion(description="x", prompt="grade", samples=3) + usage = { + "input_tokens": 100, + "output_tokens": 10, + "cache_creation_input_tokens": 7, + "cache_read_input_tokens": 900, + } + responses = [_sample_resp(0.8, usage=usage), _sample_resp(0.5, usage=usage), _sample_resp(0.6, usage=usage)] + with patch("coder_eval.criteria.llm_judge.invoke_anthropic_judge", side_effect=responses): + result = SuccessChecker(sandbox, init_registry=False, route=DirectRoute()).check(criterion) + assert isinstance(result, JudgeCriterionResult) + assert result.token_usage is not None + assert result.token_usage.uncached_input_tokens == 300 + assert result.token_usage.output_tokens == 30 + assert result.token_usage.cache_creation_input_tokens == 21 + assert result.token_usage.cache_read_input_tokens == 2700 + + +def test_judge_multi_sample_token_usage_none_when_unreported(sandbox: Sandbox) -> None: + """No sample reported usage: token_usage stays None — 'unknown' is not 'zero'.""" + from coder_eval.models import JudgeCriterionResult + + criterion = LLMJudgeCriterion(description="x", prompt="grade", samples=2) + responses = [_sample_resp(0.8), _sample_resp(0.6)] + with patch("coder_eval.criteria.llm_judge.invoke_anthropic_judge", side_effect=responses): + result = SuccessChecker(sandbox, init_registry=False, route=DirectRoute()).check(criterion) + assert isinstance(result, JudgeCriterionResult) + assert result.score == pytest.approx(0.7) + assert result.token_usage is None + + +def test_judge_multi_sample_all_fail_carries_usage_and_transcript(sandbox: Sandbox) -> None: + """The all-parse-fail result still audits: summed usage + the first sample's transcript.""" + from coder_eval.models import JudgeCriterionResult + + criterion = LLMJudgeCriterion(description="x", prompt="grade", samples=2) + usage = {"input_tokens": 40, "output_tokens": 4} + responses = [_no_verdict_resp(usage=usage), _no_verdict_resp(usage=usage)] + with patch("coder_eval.criteria.llm_judge.invoke_anthropic_judge", side_effect=responses): + result = SuccessChecker(sandbox, init_registry=False, route=DirectRoute()).check(criterion) + assert isinstance(result, JudgeCriterionResult) + assert result.score == 0.0 + assert result.error == "Judge did not call submit_verdict" + # Cost was real for both samples even though neither produced a verdict. + assert result.token_usage is not None + assert result.token_usage.uncached_input_tokens == 80 + assert result.token_usage.output_tokens == 8 + # The transcript carries the first sample's fallback marker. + assert result.transcript is not None + assert "(no verdict" in result.transcript.raw_verdict + + +def test_judge_multi_sample_runs_concurrently(sandbox: Sandbox) -> None: + """samples=N wall-clock stays ~ one sample, not N — the dispatch is concurrent. + + A sleeping fake pins the concurrency property itself: serial dispatch would + take >= samples * delay, so the assertion fails if the executor ever goes + back to a sequential loop. + """ + import time + + delay = 0.25 + + def _slow_judge(**kwargs: Any) -> dict: + time.sleep(delay) + return _sample_resp(0.5) + + criterion = LLMJudgeCriterion(description="x", prompt="grade", samples=4) + with patch("coder_eval.criteria.llm_judge.invoke_anthropic_judge", side_effect=_slow_judge): + start = time.monotonic() + result = SuccessChecker(sandbox, init_registry=False, route=DirectRoute()).check(criterion) + elapsed = time.monotonic() - start + assert result.score == 0.5 + # Serial would be >= 4 * delay = 1.0s; concurrent is ~delay. The 3x bound + # leaves generous headroom for slow CI without admitting serial dispatch. + assert elapsed < 3 * delay