Detect and quarantine wedged vGPU VFs

cmd/api/api/resources.go

@@ -87,9 +87,11 @@ func convertResourceStatus(rs resources.ResourceStatus) oapi.ResourceStatus {
 
 func convertGPUResourceStatus(gs *resources.GPUResourceStatus) oapi.GPUResourceStatus {
 	result := oapi.GPUResourceStatus{
-		Mode:       oapi.GPUResourceStatusMode(gs.Mode),
-		TotalSlots: gs.TotalSlots,
-		UsedSlots:  gs.UsedSlots,
+		Mode:             oapi.GPUResourceStatusMode(gs.Mode),
+		TotalSlots:       gs.TotalSlots,
+		UsedSlots:        gs.UsedSlots,
+		AllocatableSlots: gs.AllocatableSlots,
+		QuarantinedSlots: gs.QuarantinedSlots,
 	}
 
 	// Convert profiles (vGPU mode)

cmd/api/config/config.go

@@ -269,7 +269,8 @@ type SnapshotConfig struct {
 
 // GPUConfig holds GPU-related settings.
 type GPUConfig struct {
-	ProfileCacheTTL string `koanf:"profile_cache_ttl"`
+	ProfileCacheTTL       string `koanf:"profile_cache_ttl"`
+	VFQuarantineThreshold int    `koanf:"vf_quarantine_threshold"`
 }
 
 // Config is the top-level Hypeman server configuration.
@@ -494,7 +495,8 @@ func defaultConfig() *Config {
 		},
 
 		GPU: GPUConfig{
-			ProfileCacheTTL: "30m",
+			ProfileCacheTTL:       "30m",
+			VFQuarantineThreshold: 2,
 		},
 	}
 }
@@ -647,6 +649,9 @@ func (c *Config) Validate() error {
 	if c.Build.MaxConcurrentSourceBuilds <= 0 {
 		return fmt.Errorf("build.max_concurrent_source_builds must be positive, got %d", c.Build.MaxConcurrentSourceBuilds)
 	}
+	if c.GPU.VFQuarantineThreshold < 1 {
+		return fmt.Errorf("gpu.vf_quarantine_threshold must be >= 1, got %d", c.GPU.VFQuarantineThreshold)
+	}
 	if c.Limits.MaxConcurrentPushes <= 0 {
 		return fmt.Errorf("limits.max_concurrent_pushes must be positive, got %d", c.Limits.MaxConcurrentPushes)
 	}

cmd/api/config/config_test.go

@@ -250,6 +250,18 @@ func TestValidateRejectsInvalidMetricsPort(t *testing.T) {
 	}
 }
 
+func TestValidateRejectsInvalidVFQuarantineThreshold(t *testing.T) {
+	for _, threshold := range []int{0, -1} {
+		cfg := defaultConfig()
+		cfg.GPU.VFQuarantineThreshold = threshold
+
+		err := cfg.Validate()
+		if err == nil {
+			t.Fatalf("expected validation error for vf_quarantine_threshold %d", threshold)
+		}
+	}
+}
+
 func TestValidateRejectsInvalidMetricExportInterval(t *testing.T) {
 	cfg := defaultConfig()
 	cfg.Otel.MetricExportInterval = "not-a-duration"

cmd/api/main.go

@@ -204,6 +204,7 @@ func run() error {
 
 	// Configure GPU profile cache TTL
 	devices.SetGPUProfileCacheTTL(cfg.GPU.ProfileCacheTTL)
+	devices.SetVFQuarantineThreshold(cfg.GPU.VFQuarantineThreshold)
 
 	// Initialize OpenTelemetry (before wire initialization)
 	otelCfg := otel.Config{
@@ -653,6 +654,12 @@ func run() error {
 			return app.HealthCheckController.Run(gctx)
 		})
 	}
+	if app.VGPUSentinelController != nil {
+		grp.Go(func() error {
+			logger.Info("starting vGPU sentinel controller")
+			return app.VGPUSentinelController.Run(gctx)
+		})
+	}
 	if restartController, ok := app.InstanceManager.(interface {
 		StartRestartPolicyController(context.Context) error
 	}); ok {

cmd/api/wire.go

@@ -29,26 +29,27 @@ import (
 
 // application struct to hold initialized components
 type application struct {
-	Ctx                   context.Context
-	Logger                *slog.Logger
-	Config                *config.Config
-	ImageManager          images.Manager
-	SystemManager         system.Manager
-	NetworkManager        network.Manager
-	DeviceManager         devices.Manager
-	InstanceManager       instances.Manager
-	VolumeManager         volumes.Manager
-	BuilderManager        builders.Manager
-	IngressManager        ingress.Manager
-	BuildManager          builds.Manager
-	PushManager           imagepush.Manager
-	ResourceManager       *resources.Manager
-	GuestMemoryController guestmemory.Controller
-	AutoStandbyController *autostandby.Controller
-	HealthCheckController *instances.HealthCheckController
-	VMMetricsManager      *vm_metrics.Manager
-	Registry              *registry.Registry
-	ApiService            *api.ApiService
+	Ctx                    context.Context
+	Logger                 *slog.Logger
+	Config                 *config.Config
+	ImageManager           images.Manager
+	SystemManager          system.Manager
+	NetworkManager         network.Manager
+	DeviceManager          devices.Manager
+	InstanceManager        instances.Manager
+	VolumeManager          volumes.Manager
+	BuilderManager         builders.Manager
+	IngressManager         ingress.Manager
+	BuildManager           builds.Manager
+	PushManager            imagepush.Manager
+	ResourceManager        *resources.Manager
+	GuestMemoryController  guestmemory.Controller
+	AutoStandbyController  *autostandby.Controller
+	HealthCheckController  *instances.HealthCheckController
+	VGPUSentinelController *instances.VGPUSentinelController
+	VMMetricsManager       *vm_metrics.Manager
+	Registry               *registry.Registry
+	ApiService             *api.ApiService
 }
 
 // initializeApp is the injector function
@@ -72,6 +73,7 @@ func initializeApp() (*application, func(), error) {
 		providers.ProvideGuestMemoryController,
 		providers.ProvideAutoStandbyController,
 		providers.ProvideHealthCheckController,
+		providers.ProvideVGPUSentinelController,
 		providers.ProvideVMMetricsManager,
 		providers.ProvideRegistry,
 		api.New,

config.example.yaml

@@ -170,6 +170,12 @@ data_dir: /var/lib/hypeman
 #   idle_ttl: ""                    # delete builders idle this long (e.g. "24h");
 #                                   # destructive, empty = disabled
 
+# gpu:
+#   profile_cache_ttl: 30m          # vGPU profile metadata cache TTL
+#   vf_quarantine_threshold: 2      # distinct instance assignments that must report
+#                                   # a guest driver init failure before the VF is
+#                                   # quarantined (must be >= 1)
+
 # =============================================================================
 # Resource Limits
 # =============================================================================