ARM/CPU inference tier: vLLM on Graviton (PRD-67)

db/migrations/043_seed_cpu_instances.sql

@@ -0,0 +1,56 @@
+-- PRD-67: ARM/CPU (AWS Graviton) inference tier.
+--
+-- Adds the third accelerator class, 'cpu', alongside 'gpu' and 'neuron', and
+-- seeds Graviton3/4/5 instances. CPU rows have NO discrete accelerator:
+-- accelerator_count = 0 and accelerator_memory_gib = 0 (host DRAM, in memory_gib,
+-- is the serving memory). accelerator_name carries the Graviton generation so the
+-- UI/recommender can distinguish ISA tiers.
+--
+-- vCPU / memory verified against EC2 spec pages 2026-08-10. Graviton is 1
+-- thread/core (vcpus == physical cores). NUMA-node count is NOT AWS-documented
+-- and is single-socket (likely 1 NUMA node → TP=1); the recommender derives TP
+-- from live topology, not from this table. Graviton2 (*6g) is deliberately
+-- EXCLUDED — it lacks the SVE/I8MM/BF16 matrix extensions the Arm CPU path needs.
+
+-- 1. Expand the instance_types accelerator_type check to include 'cpu'.
+--    Idempotent: drop-and-recreate.
+ALTER TABLE instance_types DROP CONSTRAINT IF EXISTS instance_types_accelerator_type_check;
+ALTER TABLE instance_types ADD CONSTRAINT instance_types_accelerator_type_check
+    CHECK (accelerator_type = ANY (ARRAY['gpu'::text, 'neuron'::text, 'cpu'::text]));
+
+-- 2. Expand the benchmark_runs framework check to include 'vllm-cpu' (keep the
+--    existing set from migration 034). Idempotent.
+ALTER TABLE benchmark_runs DROP CONSTRAINT IF EXISTS benchmark_runs_framework_check;
+ALTER TABLE benchmark_runs ADD CONSTRAINT benchmark_runs_framework_check
+    CHECK (framework = ANY (ARRAY['vllm'::text, 'vllm-neuron'::text, 'sglang'::text, 'llm-d'::text, 'vllm-cpu'::text]));
+
+-- 3. Seed Graviton3/4/5 rows. accelerator_count=0, accelerator_memory_gib=0.
+INSERT INTO instance_types (name, family, accelerator_type, accelerator_name, accelerator_count, accelerator_memory_gib, vcpus, memory_gib) VALUES
+  -- Graviton4 (Neoverse V2, Armv9.0-a: SVE2 + I8MM + BF16). Primary target;
+  -- r8g (memory-optimized) is the natural default since CPU inference is
+  -- memory-bandwidth bound.
+  ('r8g.4xlarge',   'r8g', 'cpu', 'Graviton4', 0, 0,  16,  128),
+  ('r8g.8xlarge',   'r8g', 'cpu', 'Graviton4', 0, 0,  32,  256),
+  ('r8g.16xlarge',  'r8g', 'cpu', 'Graviton4', 0, 0,  64,  512),
+  ('r8g.24xlarge',  'r8g', 'cpu', 'Graviton4', 0, 0,  96,  768),
+  ('r8g.48xlarge',  'r8g', 'cpu', 'Graviton4', 0, 0, 192, 1536), -- large / MoE (all experts in DRAM)
+  ('m8g.12xlarge',  'm8g', 'cpu', 'Graviton4', 0, 0,  48,  192),
+  ('m8g.24xlarge',  'm8g', 'cpu', 'Graviton4', 0, 0,  96,  384), -- balanced
+  ('m8g.48xlarge',  'm8g', 'cpu', 'Graviton4', 0, 0, 192,  768),
+  ('c8g.12xlarge',  'c8g', 'cpu', 'Graviton4', 0, 0,  48,   96),
+  ('c8g.24xlarge',  'c8g', 'cpu', 'Graviton4', 0, 0,  96,  192), -- compute-leaning, tiny models
+
+  -- Graviton3 (Neoverse V1, Armv8.4-a: SVE + I8MM + BF16). Fallback — has the
+  -- matrix extensions, misses only the SVE2/Armv9 generation win.
+  ('r7g.8xlarge',   'r7g', 'cpu', 'Graviton3', 0, 0,  32,  256),
+  ('r7g.16xlarge',  'r7g', 'cpu', 'Graviton3', 0, 0,  64,  512),
+  ('m7g.16xlarge',  'm7g', 'cpu', 'Graviton3', 0, 0,  64,  256),
+  ('c7g.16xlarge',  'c7g', 'cpu', 'Graviton3', 0, 0,  64,  128),
+
+  -- Graviton5 (Neoverse V3, Armv9+). GA in us-east-2 as m9g/m9gd/c9g
+  -- (memory-optimized r9g not yet offered). Strictly >= Graviton4.
+  ('m9g.16xlarge',  'm9g', 'cpu', 'Graviton5', 0, 0,  64,  256),
+  ('m9g.24xlarge',  'm9g', 'cpu', 'Graviton5', 0, 0,  96,  384),
+  ('c9g.16xlarge',  'c9g', 'cpu', 'Graviton5', 0, 0,  64,  128)
+
+ON CONFLICT (name) DO NOTHING;

db/migrations/044_vllm_cpu_version.sql

@@ -0,0 +1,15 @@
+-- PRD-67 §10a: settable image tag for the ARM/CPU (Graviton) vLLM image,
+-- mirroring framework_version / sglang_version / pd_vllm_version (PRD-34/66).
+--
+--   vllm_cpu_version — the arm64 CPU image tag (vllm/vllm-openai-cpu:<ver>-arm64).
+--                      DISTINCT from framework_version: the CPU image is a
+--                      different repo (`-cpu`) with its own arm64 release cadence
+--                      (framework_version would ask Docker Hub for a `-cpu` tag
+--                      that may not track the GPU tag). Default = a validated
+--                      recent arm64 tag, v0.27.0 (arm64 images exist v0.18.0+;
+--                      the v0.11.2 "since" figure is for wheels, not images).
+--
+-- Defaults to the value in DefaultVLLMCPUVersion so an un-set row is
+-- byte-identical to code behavior.
+ALTER TABLE tool_versions
+    ADD COLUMN IF NOT EXISTS vllm_cpu_version TEXT NOT NULL DEFAULT 'v0.27.0';

frontend/src/api.ts

@@ -582,6 +582,7 @@ export async function putToolVersions(payload: {
   inference_perf_version: string;
   llmd_version?: string;
   pd_vllm_version?: string;
+  vllm_cpu_version?: string;
 }): Promise<ToolVersions> {
   const res = await fetch(`${BASE}/config/tool-versions`, {
     method: "PUT",

frontend/src/components/FilterBar.tsx

@@ -5,7 +5,7 @@ interface Props {
   onFilter: (filter: CatalogFilter) => void;
 }
 
-const ACCELERATORS = ["all", "gpu", "neuron"] as const;
+const ACCELERATORS = ["all", "gpu", "neuron", "cpu"] as const;
 
 export default function FilterBar({ onFilter }: Props) {
   const [model, setModel] = useState("");

frontend/src/pages/Configuration.tsx

@@ -1120,6 +1120,7 @@ function ToolVersionsCard() {
   const [inferencePerf, setInferencePerf] = useState("");
   const [llmd, setLLMD] = useState("");
   const [pdVllm, setPDVllm] = useState("");
+  const [vllmCpu, setVLLMCpu] = useState("");
   const [loading, setLoading] = useState(true);
   const [saving, setSaving] = useState(false);
   const [error, setError] = useState<string | null>(null);
@@ -1135,6 +1136,7 @@ function ToolVersionsCard() {
       setInferencePerf(fresh.inference_perf_version);
       setLLMD(fresh.llmd_version);
       setPDVllm(fresh.pd_vllm_version);
+      setVLLMCpu(fresh.vllm_cpu_version);
     } catch (err: any) {
       setError(err.message || "Failed to load tool versions");
     } finally {
@@ -1154,13 +1156,15 @@ function ToolVersionsCard() {
         inference_perf_version: inferencePerf.trim(),
         llmd_version: llmd.trim(),
         pd_vllm_version: pdVllm.trim(),
+        vllm_cpu_version: vllmCpu.trim(),
       });
       setTV(fresh);
       setFramework(fresh.framework_version);
       setSGLang(fresh.sglang_version);
       setInferencePerf(fresh.inference_perf_version);
       setLLMD(fresh.llmd_version);
       setPDVllm(fresh.pd_vllm_version);
+      setVLLMCpu(fresh.vllm_cpu_version);
       setSavedFlash(true);
       setTimeout(() => setSavedFlash(false), 2000);
     } catch (err: any) {
@@ -1176,7 +1180,8 @@ function ToolVersionsCard() {
       sglang.trim() !== tv.sglang_version ||
       inferencePerf.trim() !== tv.inference_perf_version ||
       llmd.trim() !== tv.llmd_version ||
-      pdVllm.trim() !== tv.pd_vllm_version);
+      pdVllm.trim() !== tv.pd_vllm_version ||
+      vllmCpu.trim() !== tv.vllm_cpu_version);
 
   return (
     <CollapsibleSection
@@ -1188,7 +1193,7 @@ function ToolVersionsCard() {
           {error && <span className="font-mono text-[11.5px] text-danger">{error}</span>}
           <button
             onClick={handleSave}
-            disabled={saving || loading || !dirty || !framework.trim() || !sglang.trim() || !inferencePerf.trim() || !llmd.trim() || !pdVllm.trim()}
+            disabled={saving || loading || !dirty || !framework.trim() || !sglang.trim() || !inferencePerf.trim() || !llmd.trim() || !pdVllm.trim() || !vllmCpu.trim()}
             className="btn btn-primary"
           >
             {saving ? "SAVING…" : "SAVE"}
@@ -1228,6 +1233,11 @@ function ToolVersionsCard() {
               value={pdVllm}
               onChange={setPDVllm}
             />
+            <LabeledInput
+              label="vLLM CPU Version (ARM/Graviton)"
+              value={vllmCpu}
+              onChange={setVLLMCpu}
+            />
           </div>
           <p className="meta">
             Applies to all new benchmark runs. Existing runs retain the version they were submitted with.

frontend/src/pages/Estimate.tsx

@@ -158,6 +158,7 @@ export default function Estimate() {
               <option value="">All</option>
               <option value="gpu">GPU only</option>
               <option value="neuron">Neuron only</option>
+              <option value="cpu">CPU (ARM) only</option>
             </select>
           </div>
         </div>

frontend/src/pages/ResultDetail.tsx

@@ -102,8 +102,10 @@ export default function ResultDetail() {
     return <Navigate to={`/results/${run.id}/distributed`} replace />;
   }
 
-  const isNeuron = (instanceType?.accelerator_type ?? "").toLowerCase() === "neuron";
-  const acceleratorNoun = isNeuron ? "chip" : "GPU";
+  const acceleratorType = (instanceType?.accelerator_type ?? "").toLowerCase();
+  const isNeuron = acceleratorType === "neuron";
+  const isCPU = acceleratorType === "cpu";
+  const acceleratorNoun = isNeuron ? "chip" : isCPU ? "CPU" : "GPU";
   const acceleratorCount = instanceType?.accelerator_count ?? 0;
 
   const succeeded = metrics?.successful_requests ?? 0;

frontend/src/pages/Run.tsx

@@ -432,6 +432,12 @@ export default function Run() {
   // the submit button reflects whether the user still has a path to
   // "Start Benchmark". Keep this in sync with the hostMemOnly test in
   // the warning panel below.
+  // PRD-67 §11a: gate the run form off the selected instance's accelerator type.
+  // isCPU hides GPU-only knobs (fp8 KV) and surfaces the CPU-only ones, so a user
+  // can't inadvertently set a GPU knob on a CPU run (the server also guards, §11b).
+  const selectedInstance = instanceTypes.find((t) => t.name === form.instance_type_name);
+  const isCPU = selectedInstance?.accelerator_type === "cpu";
+
   const infeasibleReason =
     recommendation?.explanation && !recommendation.explanation.feasible
       ? recommendation.explanation.reason ?? ""
@@ -440,7 +446,9 @@ export default function Run() {
     infeasibleReason !== "" &&
     /host RAM/i.test(infeasibleReason) &&
     !/VRAM|accelerator memory|divides.*heads|transformers/i.test(infeasibleReason);
-  const archInfeasible = infeasibleReason !== "" && !hostMemOnlyInfeasible;
+  // §11a: a CPU fit failure is a host-memory/bandwidth problem, not a GPU-VRAM
+  // one — treat it like host-mem-only (overridable) rather than a hard arch block.
+  const archInfeasible = infeasibleReason !== "" && !hostMemOnlyInfeasible && !isCPU;
 
   // Mirror handleSubmit's payload requirements: model + instance are
   // always required; single-mode additionally needs scenario + dataset,
@@ -676,12 +684,19 @@ export default function Run() {
               value={form.instance_type_name}
               onChange={(e) => {
                 set("instance_type_name", e.target.value);
-                const isNeuron = /^(inf|trn)/.test(e.target.value);
-                // Force neuron framework on Neuron; on GPU keep the user's
-                // current framework if they picked sglang, else default vllm.
-                if (isNeuron) {
+                // Force the accelerator-appropriate framework. Derive from the
+                // selected instance's accelerator_type (robust across the many
+                // Graviton families) rather than a name regex.
+                const at = instanceTypes.find((t) => t.name === e.target.value)?.accelerator_type;
+                if (at === "neuron") {
                   set("framework", "vllm-neuron");
-                } else if (form.framework === "vllm-neuron") {
+                } else if (at === "cpu") {
+                  set("framework", "vllm-cpu");
+                  // §11a: clear GPU-only knobs from the payload on switch to CPU so
+                  // a stale value doesn't ride along and trip the server guard.
+                  set("kv_cache_dtype", "");
+                } else if (form.framework === "vllm-neuron" || form.framework === "vllm-cpu") {
+                  // switched back to a GPU instance — reset off the forced framework.
                   set("framework", "vllm");
                 }
                 setRecommendation(null);
@@ -709,6 +724,17 @@ export default function Run() {
                     </option>
                   ))}
               </optgroup>
+              <optgroup label="CPU (ARM/Graviton)">
+                {instanceTypes
+                  .filter((t) => t.accelerator_type === "cpu")
+                  .map((t) => (
+                    // CPU has no discrete accelerator (count 0); show
+                    // vCPU/memory instead of the count×name label.
+                    <option key={t.name} value={t.name}>
+                      {t.name} ({t.accelerator_name}, {t.vcpus} vCPU / {t.memory_gib} GiB)
+                    </option>
+                  ))}
+              </optgroup>
             </select>
           </div>
           {/* Framework selector — only shown on GPU instances. Neuron is
@@ -1145,22 +1171,26 @@ export default function Run() {
               className="input w-full"
             />
           </div>
-          <div>
-            <label className="eyebrow flex items-center gap-1.5 mb-1.5">
-              KV Cache Dtype
-              <InfoTip text="Storage precision for the KV cache. fp8 halves KV-cache memory on H100/H200/L40S with negligible quality impact. The recommender sets fp8 automatically on FP8-capable GPUs. Blank/auto = match compute dtype (bf16/fp16)." />
-            </label>
-            <select
-              value={form.kv_cache_dtype}
-              onChange={(e) => set("kv_cache_dtype", e.target.value)}
-              className="input w-full"
-            >
-              <option value="">auto (match compute dtype)</option>
-              <option value="fp8">fp8</option>
-              <option value="fp8_e4m3">fp8_e4m3</option>
-              <option value="fp8_e5m2">fp8_e5m2</option>
-            </select>
-          </div>
+          {/* PRD-67 §11a: fp8 KV cache is GPU-only (CUDA/ROCm). Hide the whole
+              field on CPU so it can't be set inadvertently (the server also 400s). */}
+          {!isCPU && (
+            <div>
+              <label className="eyebrow flex items-center gap-1.5 mb-1.5">
+                KV Cache Dtype
+                <InfoTip text="Storage precision for the KV cache. fp8 halves KV-cache memory on H100/H200/L40S with negligible quality impact. The recommender sets fp8 automatically on FP8-capable GPUs. Blank/auto = match compute dtype (bf16/fp16)." />
+              </label>
+              <select
+                value={form.kv_cache_dtype}
+                onChange={(e) => set("kv_cache_dtype", e.target.value)}
+                className="input w-full"
+              >
+                <option value="">auto (match compute dtype)</option>
+                <option value="fp8">fp8</option>
+                <option value="fp8_e4m3">fp8_e4m3</option>
+                <option value="fp8_e5m2">fp8_e5m2</option>
+              </select>
+            </div>
+          )}
         </div>
 
         {/* SGLang scheduler knobs. Only shown when framework is sglang. */}

frontend/src/types.ts

@@ -777,6 +777,8 @@ export interface ToolVersions {
   // D/P image (vllm/vllm-openai) — distinct from framework_version.
   llmd_version: string;
   pd_vllm_version: string;
+  // PRD-67: ARM/CPU (Graviton) vLLM image tag (vllm/vllm-openai-cpu:<ver>-arm64).
+  vllm_cpu_version: string;
   updated_at: string;
   env_override_active: boolean;
   env_override_image?: string;
@@ -794,6 +796,9 @@ export interface ToolVersions {
   llmd_env_override_image?: string;
   pd_vllm_env_override_active: boolean;
   pd_vllm_env_override_image?: string;
+  // PRD-67: VLLM_CPU_IMAGE env var status.
+  vllm_cpu_env_override_active: boolean;
+  vllm_cpu_env_override_image?: string;
 }
 
 export interface CatalogModelEntry {

internal/api/handlers.go

@@ -444,11 +444,51 @@ func (s *Server) CreateRun(ctx context.Context, req *database.RunRequest) (strin
 	if !runtime.SupportsAccelerator(rt, instType.AcceleratorType) {
 		return "", &createRunError{http.StatusBadRequest, fmt.Sprintf("framework %q does not support %s instances", req.Framework, instType.AcceleratorType)}
 	}
+
+	// PRD-67 §11b: server-side guards for CPU runs. The UI hides these GPU-only
+	// knobs (§11a), but a direct API caller / stale UI bypasses gating, so reject
+	// them loudly here. (gpu_memory_utilization is not a request field — vLLM's
+	// default is implicit — so there's nothing to reject for it.)
+	if instType.AcceleratorType == "cpu" {
+		if req.KVCacheDtype == "fp8" {
+			return "", &createRunError{http.StatusBadRequest, "kv_cache_dtype=fp8 does not apply to CPU runs (fp8 KV cache is CUDA/ROCm only)"}
+		}
+		if req.DeploymentMode != "" && req.DeploymentMode != "single" {
+			return "", &createRunError{http.StatusBadRequest, "distributed / disaggregated runs require a GPU instance type; CPU is single-node only"}
+		}
+		// TP = NUMA-node count on CPU; single-socket Graviton is 1 NUMA node.
+		// Reject TP > NUMA count (mirrors the D/P prefill_tp>gpus-per-node guard).
+		if numa := recommend.CPUNumaNodes(instType.Name); req.TensorParallelDegree > numa {
+			return "", &createRunError{http.StatusBadRequest, fmt.Sprintf("tensor_parallel_degree (%d) exceeds the CPU instance's NUMA-node count (%d); on CPU, TP maps to NUMA nodes and intra-node parallelism comes from OMP thread binding", req.TensorParallelDegree, numa)}
+		}
+		// §11c: hard-refuse OVERSIZED DENSE UNQUANTIZED models on CPU (small bf16
+		// models are fine; the guard is about oversized unquantized). Only when the
+		// run isn't quantized — a pre-quantized/quantization-set run is exactly what
+		// we'd steer them to. Fetch model config here (CPU-only, so it doesn't add
+		// an HF call to the GPU hot path); best-effort — a fetch failure doesn't
+		// block (the recommender still refuses at recommend time).
+		unquantized := req.Quantization == nil || *req.Quantization == ""
+		if unquantized {
+			if cfg, ferr := s.FetchModelConfig(ctx, req.ModelHfID, req.HfToken); ferr == nil && cfg != nil &&
+				!cfg.PreQuantized && cfg.ParameterCount > recommend.CPUUnquantizedCeilingParams {
+				return "", &createRunError{http.StatusBadRequest, fmt.Sprintf(
+					"%.0fB-parameter model is too large to run UNQUANTIZED on CPU (ceiling ~%dB) — CPU inference is memory-bandwidth bound; use an INT8 W8A8 or INT4 W4A8 quantized checkpoint",
+					float64(cfg.ParameterCount)/1e9, recommend.CPUUnquantizedCeilingParams/1_000_000_000)}
+			}
+		}
+	}
+
 	if req.FrameworkVersion == "" {
 		if tv, _ := s.repo.GetToolVersions(ctx); tv != nil {
+			// Pass the full projection so per-accelerator runtimes resolve their
+			// own tag: vllm-cpu → VLLMCPUVersion, llm-d → LLMDVersion, etc. (a
+			// CPU run's FrameworkVersion is the vllm-openai-cpu tag, NOT the GPU
+			// framework_version — PRD-67 §10).
 			req.FrameworkVersion = rt.ResolveVersion(runtime.ToolVersions{
 				FrameworkVersion: tv.FrameworkVersion,
 				SGLangVersion:    tv.SGLangVersion,
+				LLMDVersion:      tv.LLMDVersion,
+				VLLMCPUVersion:   tv.VLLMCPUVersion,
 			})
 		}
 	}
@@ -1334,9 +1374,12 @@ func (s *Server) handleRecommend(w http.ResponseWriter, r *http.Request) {
 	}
 
 	var rec *recommend.Recommendation
-	if strings.EqualFold(instType.AcceleratorType, "neuron") {
+	switch {
+	case strings.EqualFold(instType.AcceleratorType, "neuron"):
 		rec = recommend.RecommendNeuron(*modelCfg, inst)
-	} else {
+	case strings.EqualFold(instType.AcceleratorType, "cpu"):
+		rec = recommend.RecommendCPU(*modelCfg, inst)
+	default:
 		rec = recommend.Recommend(*modelCfg, inst, allSpecs, opts)
 	}