feat(routing): add LiteLLM backend to run open-weight models on the Claude Code harness

evalboard/app/runs/[id]/[...task]/_sections.tsx

@@ -1098,7 +1098,9 @@ function ReconciliationRow({ m, unit }: { m: MessageEvent; unit: Unit }) {
                 <span className="tabular-nums text-right text-amber-700">
                     {fmtTok(m.outputTokens, "output")}
                 </span>
-                <span className="text-right text-gray-400">—</span>
+                <span className="tabular-nums text-right text-amber-700">
+                    {m.costUsd != null ? fmtUsd(m.costUsd) : "—"}
+                </span>
             </div>
         </li>
     );

evalboard/lib/__tests__/parseMessages.test.ts

@@ -769,4 +769,65 @@ describe("parseMessages — reconciliation entry", () => {
         expect(recon.parentToolUseId).toBeNull();
         expect(recon.toolUses).toEqual([]);
     });
+
+    test("prices the reconciliation row from the turn model so the Cost column sums to the turn total", () => {
+        // Mirrors the OpenRouter/LiteLLM sparse-stream case that motivated this:
+        // the per-message stream carries only output, and the bulk of the input +
+        // cache-read tokens land in the single reconciliation row. Before the fix
+        // that row read "—" for cost, so the Cost column summed to far less than
+        // the real total.
+        const turns: TurnEntry[] = [
+            {
+                model_used: "deepseek/deepseek-v4-pro",
+                messages: [
+                    {
+                        role: "assistant",
+                        model: "deepseek/deepseek-v4-pro",
+                        started_at: "2026-01-01T00:00:00.000Z",
+                        completed_at: "2026-01-01T00:00:01.000Z",
+                        generation_duration_ms: 1000,
+                        content_blocks: [{ block_type: "text", text: "hi" }],
+                        input_tokens: 0,
+                        output_tokens: 1000,
+                        cache_read_tokens: 0,
+                    },
+                    {
+                        role: "reconciliation",
+                        input_tokens: 100000,
+                        output_tokens: 0,
+                        cache_creation_tokens: 0,
+                        cache_read_tokens: 300000,
+                        note: "billed but not streamed",
+                    },
+                ],
+            },
+        ];
+        const events = parseMessages(turns);
+        const [asst, recon] = events;
+        // deepseek-v4-pro: in 0.435 / out 0.87 / cacheRead 0.003625 per MTok.
+        // reconciliation = 100000*0.435 + 300000*0.003625 = 44587.5 tok-$ / 1e6.
+        expect(recon.model).toBe("deepseek/deepseek-v4-pro");
+        expect(recon.costUsd).toBeCloseTo(0.0445875, 9);
+        // The whole point: assistant + reconciliation sum to the turn total
+        // (1000*0.87/1e6 + 0.0445875), instead of the reconciliation row reading "—".
+        expect((asst.costUsd ?? 0) + (recon.costUsd ?? 0)).toBeCloseTo(0.0454575, 9);
+    });
+
+    test("reconciliation cost stays null when the turn model is absent (legacy runs)", () => {
+        const turns: TurnEntry[] = [
+            {
+                messages: [
+                    {
+                        role: "reconciliation",
+                        input_tokens: 512,
+                        cache_creation_tokens: 1000,
+                        note: "no model",
+                    },
+                ],
+            },
+        ];
+        const recon = parseMessages(turns)[0];
+        expect(recon.model).toBeNull();
+        expect(recon.costUsd).toBeNull();
+    });
 });

evalboard/lib/__tests__/pricing-parity.test.ts

@@ -5,12 +5,17 @@ import { describe, expect, test } from "vitest";
 import { PRICING } from "../pricing";
 
 // Drift guard: lib/pricing.ts is a hand-copied mirror of the authoritative
-// Python table in src/coder_eval/pricing.py. If the backend reprices a
-// model (or adds one) and this mirror isn't updated, the frontend's "estimated"
-// USD figures silently disagree with the backend's authoritative Cost on the
-// same tokens. This test parses the Python table and asserts both tables have
-// the same model ids and the same four per-MTok rates — turning silent drift
-// into a failing build.
+// Python table in src/coder_eval/pricing.py. It exists so the frontend's
+// "estimated" USD figures agree with the backend's authoritative Cost on the
+// same tokens.
+//
+// Semantics are SUBSET, not exact-match: every model priced in lib/pricing.ts
+// must exist in pricing.py with identical rates (a frontend rate that disagrees
+// with the backend, or prices a model the backend doesn't, fails the build).
+// The frontend is NOT required to mirror every backend model — it only needs to
+// price the ones it displays, and the backend legitimately prices models the
+// evalboard never renders. (Exact-match was too strict: it forced unrelated
+// backend model additions into this file to keep the build green.)
 
 const here = dirname(fileURLToPath(import.meta.url));
 const PY_PATH = resolve(here, "../../../src/coder_eval/pricing.py");
@@ -44,20 +49,49 @@ describe("pricing.ts ↔ pricing.py parity", () => {
         expect(Object.keys(py).length).toBeGreaterThan(10);
     });
 
-    test("both tables list the same model ids", () => {
-        expect(Object.keys(PRICING).sort()).toEqual(Object.keys(py).sort());
+    test("every model in lib/pricing.ts exists in pricing.py", () => {
+        const orphans = Object.keys(PRICING).filter((m) => !(m in py));
+        expect(
+            orphans,
+            `priced in lib/pricing.ts but absent from pricing.py: ${orphans.join(", ")}`,
+        ).toEqual([]);
     });
 
-    test("every model has identical input/output/cacheWrite/cacheRead rates", () => {
-        for (const [model, [input, output, cw, cr]] of Object.entries(py)) {
-            const ts = PRICING[model];
-            expect(ts, `missing in lib/pricing.ts: ${model}`).toBeDefined();
+    test("shared models have identical input/output/cacheWrite/cacheRead rates", () => {
+        for (const [model, ts] of Object.entries(PRICING)) {
+            const rates = py[model];
+            expect(rates, `not priced in pricing.py: ${model}`).toBeDefined();
             expect([
                 ts.inputPerMTok,
                 ts.outputPerMTok,
                 ts.cacheWritePerMTok,
                 ts.cacheReadPerMTok,
-            ]).toEqual([input, output, cw, cr]);
+            ]).toEqual(rates);
         }
     });
+
+    // Python-priced models we deliberately do NOT mirror to the frontend: heavy
+    // frontier Claude/GPT variants the evalboard never runs, so pricing them here
+    // adds nothing. Kept explicit (not a blanket "ignore extras") so a NEW model
+    // added to pricing.py that ISN'T here and ISN'T in PRICING breaks the build —
+    // catching a real litellm-relevant omission (e.g. the Bedrock open-weight ids
+    // that previously rendered "—" for cost).
+    const DELIBERATELY_UNMIRRORED = new Set([
+        "claude-sonnet-5",
+        "gpt-5.4-mini",
+        "gpt-5.4-nano",
+        "gpt-5.4-pro",
+        "gpt-5.5-pro",
+        "gpt-5.6-sol",
+        "gpt-5.6-terra",
+        "gpt-5.6-luna",
+    ]);
+
+    test("every pricing.py model is mirrored in pricing.ts or explicitly unmirrored", () => {
+        const missing = Object.keys(py).filter((m) => !(m in PRICING) && !DELIBERATELY_UNMIRRORED.has(m));
+        expect(
+            missing,
+            `priced in pricing.py but missing from pricing.ts — mirror it or add to DELIBERATELY_UNMIRRORED: ${missing.join(", ")}`,
+        ).toEqual([]);
+    });
 });

evalboard/lib/__tests__/pricing.test.ts

@@ -38,6 +38,14 @@ describe("resolvePricing", () => {
     test("knows the current default opus id", () => {
         expect(resolvePricing("claude-opus-4-8")?.outputPerMTok).toBe(75);
     });
+
+    test("strips LiteLLM/Bedrock routing + region prefixes (recorded model_used is qualified)", () => {
+        // The recorded model arrives prefixed on litellm/Bedrock runs; without the
+        // strip these rendered "—" for the whole cost column.
+        expect(resolvePricing("converse/zai.glm-5")?.outputPerMTok).toBe(3.84);
+        expect(resolvePricing("bedrock/converse/deepseek.v3.2")?.inputPerMTok).toBe(0.74);
+        expect(resolvePricing("eu.anthropic.claude-sonnet-4-6")?.outputPerMTok).toBe(15);
+    });
 });
 
 describe("tokenBucketUsd", () => {

evalboard/lib/pricing.ts

@@ -50,6 +50,20 @@ export const PRICING: Record<string, Pricing> = {
     "gemini-3.1-pro-preview-customtools": p(2, 12, 2, 0.2),
     "gemini-3.5-flash": p(1.5, 9, 1.5, 0.15),
     "gemini-3-flash-preview": p(1.5, 9, 1.5, 0.15),
+    // OpenRouter open-weight models (litellm backend). Mirror of pricing.py;
+    // these providers cache implicitly (cache_write == input, unused) so only
+    // cache_read carries a discounted rate. NOTE: OpenRouter routes per-request,
+    // so these rates are only accurate when the litellm config pins the provider
+    // (sort: price) — otherwise the billed rate can differ from the headline.
+    "moonshotai/kimi-k3": p(3, 15, 3, 0.3),
+    "z-ai/glm-5.2": p(0.826, 2.596, 0.826, 0.1534),
+    "deepseek/deepseek-v4-pro": p(0.435, 0.87, 0.435, 0.003625),
+    // Bedrock open-weight models (litellm backend, eu-north-1). Mirror of pricing.py.
+    // The recorded model_used arrives prefixed (e.g. "converse/zai.glm-5"), so
+    // resolvePricing strips the routing/region prefixes before lookup.
+    "deepseek.v3.2": p(0.74, 2.22, 0.74, 0),
+    "zai.glm-5": p(1.2, 3.84, 1.2, 0),
+    "moonshotai.kimi-k2.5": p(0.72, 3.6, 0.72, 0),
 };
 
 function p(
@@ -66,21 +80,45 @@ function p(
     };
 }
 
-// Resolve pricing for a model id, tolerating undated aliases (the recorded
-// model is usually the canonical id like "claude-sonnet-4-6", but be lenient
-// about a trailing date suffix). Matches the Python source's exact-match
-// semantics, plus a date-suffix strip — deliberately NO loose prefix match: a
-// substring fallback would silently price `gpt-5-mini` at full `gpt-5` rates,
-// presenting a multi-x overcharge as an authoritative-looking figure. Unknown
-// ids return null (render "—") rather than a wrong number.
+// Strip the LiteLLM/Bedrock routing + region/vendor prefixes back to the bare
+// pricing key — mirror of src/coder_eval/pricing.py::_normalize_model, since the
+// recorded model_used arrives qualified (e.g. "converse/zai.glm-5",
+// "eu.anthropic.claude-sonnet-4-6"). Idempotent on already-bare ids.
+const _ROUTING_PREFIXES = ["bedrock/converse/", "bedrock/", "converse/"];
+const _REGION_PREFIXES = ["eu.", "us.", "apac.", "global."];
+function normalizeModel(model: string): string {
+    let m = model.trim();
+    for (const pre of _ROUTING_PREFIXES) {
+        if (m.startsWith(pre)) {
+            m = m.slice(pre.length);
+            break;
+        }
+    }
+    for (const pre of _REGION_PREFIXES) {
+        if (m.startsWith(pre)) {
+            m = m.slice(pre.length);
+            break;
+        }
+    }
+    if (m.startsWith("anthropic.")) m = m.slice("anthropic.".length);
+    return m;
+}
+
+// Resolve pricing for a model id, tolerating routing/region prefixes and undated
+// aliases (the recorded model is usually the canonical id like "claude-sonnet-4-6",
+// but LiteLLM/Bedrock runs record it prefixed, and some carry a trailing date).
+// Deliberately NO loose *substring* match: that would silently price `gpt-5-mini`
+// at full `gpt-5` rates, presenting a multi-x overcharge as an authoritative-looking
+// figure. Unknown ids return null (render "—") rather than a wrong number.
 //
 // Object.hasOwn (not `PRICING[model]` truthiness) guards against a degenerate
 // id like "constructor"/"toString" resolving to an inherited prototype member.
 export function resolvePricing(model: string | null): Pricing | null {
     if (!model) return null;
-    if (Object.hasOwn(PRICING, model)) return PRICING[model];
+    const norm = normalizeModel(model);
+    if (Object.hasOwn(PRICING, norm)) return PRICING[norm];
     // Try stripping a trailing -YYYYMMDD date.
-    const undated = model.replace(/-\d{8}$/, "");
+    const undated = norm.replace(/-\d{8}$/, "");
     if (Object.hasOwn(PRICING, undated)) return PRICING[undated];
     return null;
 }

evalboard/lib/runs.ts

@@ -1198,6 +1198,9 @@ interface MessageEntry {
 export interface TurnEntry {
     commands?: CommandEntry[];
     messages?: MessageEntry[];
+    // Model the turn ran on (iteration `model_used`). Used to price the synthetic
+    // reconciliation row, which carries no model of its own.
+    model_used?: string | null;
     token_usage?: TokenUsageEntry | null;
     result_summary?: {
         result?: string | null;
@@ -1716,8 +1719,23 @@ export function parseMessages(turns: TurnEntry[]): MessageEvent[] {
                 reasoningTokens: null,
                 thinkingOutputTokens: null,
                 textOutputTokens: null,
-                model: null,
-                costUsd: null,
+                // Price the reconciliation row so summing the Cost column across
+                // the stream reproduces the turn total. Critical for providers
+                // (e.g. OpenRouter/LiteLLM) whose per-message stream is sparse and
+                // dumps most tokens into this one row — leaving it uncosted made
+                // the visible per-row costs sum to far less than the real total.
+                // Excluded from the cost simulator (by role), and the authoritative
+                // task total_cost_usd is unaffected (it reads the backend aggregate,
+                // not a sum of per-message costUsd), so there is no double-count.
+                model: turn.model_used ?? null,
+                costUsd: messageCostUsd({
+                    model: turn.model_used ?? null,
+                    inputTokens: typeof msg.input_tokens === "number" ? msg.input_tokens : null,
+                    outputTokens: typeof msg.output_tokens === "number" ? msg.output_tokens : null,
+                    cacheWriteTokens:
+                        typeof msg.cache_creation_tokens === "number" ? msg.cache_creation_tokens : null,
+                    cacheReadTokens: typeof msg.cache_read_tokens === "number" ? msg.cache_read_tokens : null,
+                }),
                 note: typeof msg.note === "string" ? msg.note : null,
             });
         }