diff --git a/cmd/api/api/resources.go b/cmd/api/api/resources.go index ebb6ad951..dec9f35eb 100644 --- a/cmd/api/api/resources.go +++ b/cmd/api/api/resources.go @@ -87,9 +87,11 @@ func convertResourceStatus(rs resources.ResourceStatus) oapi.ResourceStatus { func convertGPUResourceStatus(gs *resources.GPUResourceStatus) oapi.GPUResourceStatus { result := oapi.GPUResourceStatus{ - Mode: oapi.GPUResourceStatusMode(gs.Mode), - TotalSlots: gs.TotalSlots, - UsedSlots: gs.UsedSlots, + Mode: oapi.GPUResourceStatusMode(gs.Mode), + TotalSlots: gs.TotalSlots, + UsedSlots: gs.UsedSlots, + AllocatableSlots: gs.AllocatableSlots, + QuarantinedSlots: gs.QuarantinedSlots, } // Convert profiles (vGPU mode) diff --git a/cmd/api/config/config.go b/cmd/api/config/config.go index b46f4f2e5..c250a3621 100644 --- a/cmd/api/config/config.go +++ b/cmd/api/config/config.go @@ -269,7 +269,8 @@ type SnapshotConfig struct { // GPUConfig holds GPU-related settings. type GPUConfig struct { - ProfileCacheTTL string `koanf:"profile_cache_ttl"` + ProfileCacheTTL string `koanf:"profile_cache_ttl"` + VFQuarantineThreshold int `koanf:"vf_quarantine_threshold"` } // Config is the top-level Hypeman server configuration. @@ -494,7 +495,8 @@ func defaultConfig() *Config { }, GPU: GPUConfig{ - ProfileCacheTTL: "30m", + ProfileCacheTTL: "30m", + VFQuarantineThreshold: 2, }, } } @@ -647,6 +649,9 @@ func (c *Config) Validate() error { if c.Build.MaxConcurrentSourceBuilds <= 0 { return fmt.Errorf("build.max_concurrent_source_builds must be positive, got %d", c.Build.MaxConcurrentSourceBuilds) } + if c.GPU.VFQuarantineThreshold < 1 { + return fmt.Errorf("gpu.vf_quarantine_threshold must be >= 1, got %d", c.GPU.VFQuarantineThreshold) + } if c.Limits.MaxConcurrentPushes <= 0 { return fmt.Errorf("limits.max_concurrent_pushes must be positive, got %d", c.Limits.MaxConcurrentPushes) } diff --git a/cmd/api/config/config_test.go b/cmd/api/config/config_test.go index 5660d878e..efd52f208 100644 --- a/cmd/api/config/config_test.go +++ b/cmd/api/config/config_test.go @@ -250,6 +250,18 @@ func TestValidateRejectsInvalidMetricsPort(t *testing.T) { } } +func TestValidateRejectsInvalidVFQuarantineThreshold(t *testing.T) { + for _, threshold := range []int{0, -1} { + cfg := defaultConfig() + cfg.GPU.VFQuarantineThreshold = threshold + + err := cfg.Validate() + if err == nil { + t.Fatalf("expected validation error for vf_quarantine_threshold %d", threshold) + } + } +} + func TestValidateRejectsInvalidMetricExportInterval(t *testing.T) { cfg := defaultConfig() cfg.Otel.MetricExportInterval = "not-a-duration" diff --git a/cmd/api/main.go b/cmd/api/main.go index faa9ac15a..5d78954d8 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -204,6 +204,7 @@ func run() error { // Configure GPU profile cache TTL devices.SetGPUProfileCacheTTL(cfg.GPU.ProfileCacheTTL) + devices.SetVFQuarantineThreshold(cfg.GPU.VFQuarantineThreshold) // Initialize OpenTelemetry (before wire initialization) otelCfg := otel.Config{ @@ -653,6 +654,12 @@ func run() error { return app.HealthCheckController.Run(gctx) }) } + if app.VGPUSentinelController != nil { + grp.Go(func() error { + logger.Info("starting vGPU sentinel controller") + return app.VGPUSentinelController.Run(gctx) + }) + } if restartController, ok := app.InstanceManager.(interface { StartRestartPolicyController(context.Context) error }); ok { diff --git a/cmd/api/wire.go b/cmd/api/wire.go index 133bf41f6..123d59bf5 100644 --- a/cmd/api/wire.go +++ b/cmd/api/wire.go @@ -29,26 +29,27 @@ import ( // application struct to hold initialized components type application struct { - Ctx context.Context - Logger *slog.Logger - Config *config.Config - ImageManager images.Manager - SystemManager system.Manager - NetworkManager network.Manager - DeviceManager devices.Manager - InstanceManager instances.Manager - VolumeManager volumes.Manager - BuilderManager builders.Manager - IngressManager ingress.Manager - BuildManager builds.Manager - PushManager imagepush.Manager - ResourceManager *resources.Manager - GuestMemoryController guestmemory.Controller - AutoStandbyController *autostandby.Controller - HealthCheckController *instances.HealthCheckController - VMMetricsManager *vm_metrics.Manager - Registry *registry.Registry - ApiService *api.ApiService + Ctx context.Context + Logger *slog.Logger + Config *config.Config + ImageManager images.Manager + SystemManager system.Manager + NetworkManager network.Manager + DeviceManager devices.Manager + InstanceManager instances.Manager + VolumeManager volumes.Manager + BuilderManager builders.Manager + IngressManager ingress.Manager + BuildManager builds.Manager + PushManager imagepush.Manager + ResourceManager *resources.Manager + GuestMemoryController guestmemory.Controller + AutoStandbyController *autostandby.Controller + HealthCheckController *instances.HealthCheckController + VGPUSentinelController *instances.VGPUSentinelController + VMMetricsManager *vm_metrics.Manager + Registry *registry.Registry + ApiService *api.ApiService } // initializeApp is the injector function @@ -72,6 +73,7 @@ func initializeApp() (*application, func(), error) { providers.ProvideGuestMemoryController, providers.ProvideAutoStandbyController, providers.ProvideHealthCheckController, + providers.ProvideVGPUSentinelController, providers.ProvideVMMetricsManager, providers.ProvideRegistry, api.New, diff --git a/cmd/api/wire_gen.go b/cmd/api/wire_gen.go index 9eb13c6ea..57034551a 100644 --- a/cmd/api/wire_gen.go +++ b/cmd/api/wire_gen.go @@ -82,6 +82,10 @@ func initializeApp() (*application, func(), error) { } autostandbyController := providers.ProvideAutoStandbyController(instancesManager, config, logger) healthCheckController := providers.ProvideHealthCheckController(instancesManager, logger) + vgpuSentinelController, err := providers.ProvideVGPUSentinelController(instancesManager, logger) + if err != nil { + return nil, nil, err + } vm_metricsManager, err := providers.ProvideVMMetricsManager(instancesManager, config, logger) if err != nil { return nil, nil, err @@ -92,26 +96,27 @@ func initializeApp() (*application, func(), error) { } apiService := api.New(config, manager, instancesManager, volumesManager, buildersManager, networkManager, devicesManager, ingressManager, buildsManager, imagepushManager, resourcesManager, controller, autostandbyController, vm_metricsManager) mainApplication := &application{ - Ctx: context, - Logger: logger, - Config: config, - ImageManager: manager, - SystemManager: systemManager, - NetworkManager: networkManager, - DeviceManager: devicesManager, - InstanceManager: instancesManager, - VolumeManager: volumesManager, - BuilderManager: buildersManager, - IngressManager: ingressManager, - BuildManager: buildsManager, - PushManager: imagepushManager, - ResourceManager: resourcesManager, - GuestMemoryController: controller, - AutoStandbyController: autostandbyController, - HealthCheckController: healthCheckController, - VMMetricsManager: vm_metricsManager, - Registry: registry, - ApiService: apiService, + Ctx: context, + Logger: logger, + Config: config, + ImageManager: manager, + SystemManager: systemManager, + NetworkManager: networkManager, + DeviceManager: devicesManager, + InstanceManager: instancesManager, + VolumeManager: volumesManager, + BuilderManager: buildersManager, + IngressManager: ingressManager, + BuildManager: buildsManager, + PushManager: imagepushManager, + ResourceManager: resourcesManager, + GuestMemoryController: controller, + AutoStandbyController: autostandbyController, + HealthCheckController: healthCheckController, + VGPUSentinelController: vgpuSentinelController, + VMMetricsManager: vm_metricsManager, + Registry: registry, + ApiService: apiService, } return mainApplication, func() { }, nil @@ -121,24 +126,25 @@ func initializeApp() (*application, func(), error) { // application struct to hold initialized components type application struct { - Ctx context.Context - Logger *slog.Logger - Config *config.Config - ImageManager images.Manager - SystemManager system.Manager - NetworkManager network.Manager - DeviceManager devices.Manager - InstanceManager instances.Manager - VolumeManager volumes.Manager - BuilderManager builders.Manager - IngressManager ingress.Manager - BuildManager builds.Manager - PushManager imagepush.Manager - ResourceManager *resources.Manager - GuestMemoryController guestmemory.Controller - AutoStandbyController *autostandby.Controller - HealthCheckController *instances.HealthCheckController - VMMetricsManager *vm_metrics.Manager - Registry *registry.Registry - ApiService *api.ApiService + Ctx context.Context + Logger *slog.Logger + Config *config.Config + ImageManager images.Manager + SystemManager system.Manager + NetworkManager network.Manager + DeviceManager devices.Manager + InstanceManager instances.Manager + VolumeManager volumes.Manager + BuilderManager builders.Manager + IngressManager ingress.Manager + BuildManager builds.Manager + PushManager imagepush.Manager + ResourceManager *resources.Manager + GuestMemoryController guestmemory.Controller + AutoStandbyController *autostandby.Controller + HealthCheckController *instances.HealthCheckController + VGPUSentinelController *instances.VGPUSentinelController + VMMetricsManager *vm_metrics.Manager + Registry *registry.Registry + ApiService *api.ApiService } diff --git a/config.example.yaml b/config.example.yaml index ebef41257..70d55fa68 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -170,6 +170,12 @@ data_dir: /var/lib/hypeman # idle_ttl: "" # delete builders idle this long (e.g. "24h"); # # destructive, empty = disabled +# gpu: +# profile_cache_ttl: 30m # vGPU profile metadata cache TTL +# vf_quarantine_threshold: 2 # distinct instance assignments that must report +# # a guest driver init failure before the VF is +# # quarantined (must be >= 1) + # ============================================================================= # Resource Limits # ============================================================================= diff --git a/lib/devices/GPU.md b/lib/devices/GPU.md index d04dcf599..f039afa1c 100644 --- a/lib/devices/GPU.md +++ b/lib/devices/GPU.md @@ -49,8 +49,10 @@ curl -s http://localhost:4973/resources | jq .gpu "mode": "vgpu", "total_slots": 64, "used_slots": 5, + "allocatable_slots": 57, + "quarantined_slots": 2, "profiles": [ - {"name": "L40S-1Q", "framebuffer_mb": 1024, "available": 59}, + {"name": "L40S-1Q", "framebuffer_mb": 1024, "available": 57}, {"name": "L40S-2Q", "framebuffer_mb": 2048, "available": 30}, {"name": "L40S-4Q", "framebuffer_mb": 4096, "available": 16} ] @@ -97,7 +99,7 @@ Instance Create → Assign profile to VF → Attach VF to VM → Instance Runnin Instance Stop/Delete → Release profile → VF available again ``` -Hypeman reconciles orphaned assignments with a periodic fail-closed pass: once at startup and every minute afterward (skipped entirely on hosts without GPUs). Each pass releases assignments whose owning instance is no longer live and clears their metadata, then sweeps device-level leftovers with no live metadata claim. Devices held open by a running VMM and assignments younger than five minutes are preserved, so a release that fails during stop or delete (typically because a GPU-busy VMM's kernel-side VFIO teardown outlives the force-kill wait) is simply retried on later passes until the device is free. +Hypeman reconciles orphaned assignments with a periodic fail-closed pass: once at startup and every minute afterward (skipped entirely on hosts without GPUs). Each pass releases assignments whose owning instance is no longer live and clears their metadata, then sweeps device-level leftovers with no live metadata claim. Devices held open by a running VMM and assignments younger than five minutes are preserved, so a release that fails during stop or delete (typically because a GPU-busy VMM's kernel-side VFIO teardown outlives the force-kill wait) is simply retried on later passes until the device is free. An ambiguous hypervisor ownership check also preserves the assignment, logs a warning, and increments `hypeman_instances_vgpu_reconcile_liveness_uncertain_total`. ### Hypervisor Support @@ -121,6 +123,8 @@ curl -s http://localhost:4973/resources | jq .gpu "mode": "passthrough", "total_slots": 4, "used_slots": 2, + "allocatable_slots": 2, + "quarantined_slots": 0, "devices": [ {"name": "NVIDIA L40S", "available": true}, {"name": "NVIDIA L40S", "available": false} @@ -185,8 +189,10 @@ Returns GPU status along with other resources: "mode": "vgpu", "total_slots": 64, "used_slots": 5, + "allocatable_slots": 57, + "quarantined_slots": 2, "profiles": [ - {"name": "L40S-1Q", "framebuffer_mb": 1024, "available": 59} + {"name": "L40S-1Q", "framebuffer_mb": 1024, "available": 57} ] } } @@ -282,13 +288,49 @@ NVRM: GPU 0000:00:03.0: RmInitAdapter failed! (0x22:0x65:884) ``` (0x65 = timeout; the guest's init requests are never answered, and -`/proc/interrupts` shows the GPU's MSI-X vectors allocated but idle). Because -placement is deterministic least-loaded, an idle host re-picks the same VF for -every request, so one wedged VF presents as all vGPU instances failing while -`/resources` reports full capacity. - -The wedge itself leaves no host-side log: no kernel error, no XID, no plugin -crash. The trigger is a SIGKILL delivered to QEMU while the vGPU plugin is +`/proc/interrupts` shows the GPU's MSI-X vectors allocated but idle). + +Hypeman detects this automatically: the guest agent watches the guest kernel +log (`/dev/kmsg`) for that line and reports it as a `HYPEMAN-GPU-INIT-FAILED` +marker in the instance's `logs/app.log`, which the vGPU sentinel controller +scans for every vendor VFIO instance. Each match records one init failure +against the VF in `/gpu/vf-health.json` (it survives restarts), +tallied per instance assignment; once failures accumulate from +`gpu.vf_quarantine_threshold` distinct assignments (default 2), the VF is +quarantined: excluded from placement and from advertised profile +availability, and its parent GPU becomes overflow-only — deprioritized for +new placements. The guest agent also probes driver init at boot with +`nvidia-smi -L` (when present in the image): the device open runs +RmInitAdapter, so on a wedged VF the probe itself triggers the failure line +without waiting for the workload to touch the GPU. On success it emits a +terminal `HYPEMAN-GPU-INIT-OK` marker and suppresses later failure reports. +The marker clears failures through that assignment; if the assignment's +transient failure crossed the threshold, its later success also rescinds that +quarantine. Other quarantines require manual recovery. + +`used_slots` includes quarantined VFs still held by running instances, so it +can overlap `quarantined_slots`; use `allocatable_slots` for admission. + +Below-threshold failures log at warn and increment +`hypeman_instances_vgpu_sentinel_init_failures_total`; quarantines log at +error and increment `hypeman_instances_vgpu_sentinel_quarantines_total`. +`hypeman_instances_vgpu_quarantined_vfs` gauges the current count. A systemic +guest/host driver mismatch can still quarantine every VF, so validate driver +changes on a test host and alert on the failure counter. + +Detection requires the hypeman guest agent. Start archives the previous +boot's app log before persisting a new assignment, so a report racing a +stop/start may be deferred until the next victim boot. Only a complete, +standalone guest-agent marker matches; ordinary output containing the token +does not. A root guest can still forge the full serial-console line across +enough assignments to quarantine its VFs, but quarantine only removes +capacity and never touches an instance. + +The wedge-creating kill itself leaves no host-side log: no kernel error, no +XID, no plugin crash. Detection therefore happens on the next boot that lands +on the VF, whose guest driver starts failing ~27s after spawn. + +The trigger is a SIGKILL delivered to QEMU while the vGPU plugin is still initializing the VF (roughly the first seconds after process start): a single hard kill in that window wedges the VF near-deterministically, while QEMU processes that exit voluntarily — error exits, QMP quit, SIGTERM — @@ -303,18 +345,46 @@ External SIGKILLs (OOM killer, manual `kill -9`) can still trigger it. Confirm by assigning the same profile on a different VF: if that guest initializes, the VF is wedged, not the driver stack. Remediate by cycling SR-IOV on the parent GPU (this destroys and recreates all of its VFs, so it -requires no vGPU assignments on that GPU): +requires no vGPU assignments on that GPU). The DCGM quiesce is not optional: +with `nv-hostengine`/`dcgm-exporter` holding the GPUs open, `sriov-manage -d` +fails with `Cannot obtain unbindLock` on first contact. + +Any manual edit to `vf-health.json` needs an immediate hypeman restart: the +store loads only at startup, and a failure report landing first re-persists +the in-memory set over your edit. The restart does not disturb running VMs — +startup reconciliation protects live VFs. + +**Draining the parent GPU.** Overflow-only is a preference, not a cordon: +under capacity pressure new placements still land on the card's healthy VFs +and refill it. To drain the card, quarantine all of its VFs by hand — add +records to the versioned `vf-health.json` (`{"version": 1, "records": +[{"vf_address": "...", "quarantined_at": "..."}]}`) and restart. Running +instances are untouched and +drain through their normal lifecycle: standby is blocked for vGPU instances, +so only a running VM pins a VF, and each stop or delete frees one for good. +Monitor by listing instances whose `gpu.device_path` sits under the parent +GPU; once none remain, run the cycle below. ```bash +# 1. Quiesce the services holding the GPU (required for the unbind lock). +systemctl stop nvidia-dcgm-exporter nvidia-dcgm + +# 2. Cycle SR-IOV on the parent GPU. /usr/lib/nvidia/sriov-manage -d /usr/lib/nvidia/sriov-manage -e + +# 3. Restart the quiesced services. +systemctl start nvidia-dcgm nvidia-dcgm-exporter ``` +After the cycle, remove the card's entries from `vf-health.json`, restart, +and boot a GPU instance to verify recovery. If the cycle did not work, the +sentinel quarantines the VF again after the configured number of fresh +assignment failures. + Do not unbind/rebind the VF from the nvidia driver — it breaks the nvidia-vgpu-vfio core-device registration (`vfio_pci_core_device not found`) and the VF stops accepting assignments entirely until the SR-IOV cycle. -Services holding the GPU (DCGM, persistenced) must be stopped for the cycle -to obtain the unbind lock. ### vGPU assignment fails diff --git a/lib/devices/manager.go b/lib/devices/manager.go index 30763c04d..6b9f6340a 100644 --- a/lib/devices/manager.go +++ b/lib/devices/manager.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "log/slog" "os" "runtime" "strings" @@ -85,6 +86,9 @@ type manager struct { // NewManager creates a new device manager. // Use SetLivenessChecker after construction to enable accurate orphan detection. func NewManager(p *paths.Paths) Manager { + if err := initVFHealth(p.VFHealthState()); err != nil { + slog.Default().Error("failed to load VF health state; vGPU placement is disabled until the state file is repaired or removed", "error", err) + } return &manager{ paths: p, vfioBinder: NewVFIOBinder(), diff --git a/lib/devices/vendor_vfio_linux.go b/lib/devices/vendor_vfio_linux.go index a1378a3c7..b923439a3 100644 --- a/lib/devices/vendor_vfio_linux.go +++ b/lib/devices/vendor_vfio_linux.go @@ -8,12 +8,12 @@ import ( "fmt" "log/slog" "maps" + "math/rand/v2" "os" "path/filepath" "sort" "strconv" "strings" - "sync" "syscall" "time" @@ -37,18 +37,16 @@ type vendorVFIOSysfs struct { owners map[string]vendorVFIOOwner framebufferByType map[string]int openVFIOPathsFunc func() (map[string]struct{}, error) + pickVFIndex func(n int) int } -var ( - hostVendorVFIO = vendorVFIOSysfs{ - pciDevicesPath: pciDevicesPath, - procPath: procPath, - vfioDevicesPath: vfioDevicesPath, - owners: make(map[string]vendorVFIOOwner), - framebufferByType: make(map[string]int), - } - vendorVFIOMu sync.Mutex -) +var hostVendorVFIO = vendorVFIOSysfs{ + pciDevicesPath: pciDevicesPath, + procPath: procPath, + vfioDevicesPath: vfioDevicesPath, + owners: make(map[string]vendorVFIOOwner), + framebufferByType: make(map[string]int), +} func (s vendorVFIOSysfs) discoverVFs() ([]VirtualFunction, error) { entries, err := os.ReadDir(s.pciDevicesPath) @@ -110,6 +108,10 @@ func (s vendorVFIOSysfs) discoverVFs() ([]VirtualFunction, error) { // available_instances. This is a best-effort snapshot because creating on one // VF may revoke the type from siblings that share its GPU framebuffer. func (s vendorVFIOSysfs) listProfiles(vfs []VirtualFunction) ([]GPUProfile, error) { + quarantined, err := vfHealth.checkedAddresses() + if err != nil { + return nil, err + } profilesByType := make(map[string]profileMetadata) creatableVFs := make(map[string]int) for _, vf := range vfs { @@ -121,9 +123,10 @@ func (s vendorVFIOSysfs) listProfiles(vfs []VirtualFunction) ([]GPUProfile, erro slog.Default().Warn("skipping unreadable creatable vGPU types", "vf", vf.PCIAddress, "error", err) continue } + _, bad := quarantined[vf.PCIAddress] for _, profile := range creatable { profilesByType[profile.TypeName] = profile - if !vf.Allocated { + if !vf.Allocated && !bad { creatableVFs[profile.TypeName]++ } } @@ -315,10 +318,22 @@ func (s vendorVFIOSysfs) reconcile(ctx context.Context, protectedDevicePaths map } func (s vendorVFIOSysfs) selectLeastLoadedVF(vfs []VirtualFunction, profileType string) (string, error) { + quarantined, err := vfHealth.checkedAddresses() + if err != nil { + return "", err + } usageByGPU := make(map[string]int) unknownUsageByGPU := make(map[string]bool) + quarantinedByGPU := make(map[string]int) freeByGPU := make(map[string][]VirtualFunction) for _, vf := range vfs { + _, bad := quarantined[vf.PCIAddress] + if bad { + quarantinedByGPU[vf.ParentGPU]++ + if !vf.Allocated { + continue + } + } if vf.Allocated { // framebufferByType only covers currently creatable profiles, so // after a restart an allocated type can be missing when its @@ -352,6 +367,9 @@ func (s vendorVFIOSysfs) selectLeastLoadedVF(vfs []VirtualFunction, profileType gpus = append(gpus, gpu) } sort.Slice(gpus, func(i, j int) bool { + if quarantinedByGPU[gpus[i]] != quarantinedByGPU[gpus[j]] { + return quarantinedByGPU[gpus[i]] < quarantinedByGPU[gpus[j]] + } if unknownUsageByGPU[gpus[i]] != unknownUsageByGPU[gpus[j]] { return !unknownUsageByGPU[gpus[i]] } @@ -363,7 +381,12 @@ func (s vendorVFIOSysfs) selectLeastLoadedVF(vfs []VirtualFunction, profileType if len(gpus) == 0 { return "", nil } - return freeByGPU[gpus[0]][0].PCIAddress, nil + candidates := freeByGPU[gpus[0]] + pick := s.pickVFIndex + if pick == nil { + pick = rand.IntN + } + return candidates[pick(len(candidates))].PCIAddress, nil } func (s vendorVFIOSysfs) profileMetadata(vfs []VirtualFunction) ([]profileMetadata, error) { diff --git a/lib/devices/vendor_vfio_linux_test.go b/lib/devices/vendor_vfio_linux_test.go index 098d016e3..a65fd0a88 100644 --- a/lib/devices/vendor_vfio_linux_test.go +++ b/lib/devices/vendor_vfio_linux_test.go @@ -621,3 +621,69 @@ func assertFileValue(t *testing.T, path, expected string) { require.NoError(t, err) assert.Equal(t, expected, string(value)) } + +func TestVendorVFIOSkipsQuarantinedVF(t *testing.T) { + resetVFHealthStore(t) + quarantineVF(t, "0000:82:00.4") + + sysfs := newTestVendorVFIOSysfs(t) + sysfs.pickVFIndex = func(int) int { return 0 } + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "0", testCreatableTypes) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.5", "43", "0", testCreatableTypes) + + device, err := sysfs.create(context.Background(), "NVIDIA L40S-1Q", "instance-1") + require.NoError(t, err) + assert.Equal(t, "0000:82:00.5", device.VFAddress) +} + +func TestVendorVFIONoVFWhenAllQuarantined(t *testing.T) { + resetVFHealthStore(t) + quarantineVF(t, "0000:82:00.4") + + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "0", testCreatableTypes) + + _, err := sysfs.create(context.Background(), "NVIDIA L40S-1Q", "instance-1") + require.ErrorContains(t, err, "no available VF") +} + +func TestVendorVFIOCardBiasAvoidsGPUWithQuarantinedVF(t *testing.T) { + resetVFHealthStore(t) + quarantineVF(t, "0000:82:00.4") + + sysfs := newTestVendorVFIOSysfs(t) + sysfs.pickVFIndex = func(int) int { return 0 } + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "0", testCreatableTypes) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.5", "43", "0", testCreatableTypes) + sysfs.addVF(t, "0000:e3:00.0", "0000:e3:00.4", "44", "0", testCreatableTypes) + + device, err := sysfs.create(context.Background(), "NVIDIA L40S-1Q", "instance-1") + require.NoError(t, err) + assert.Equal(t, "0000:e3:00.4", device.VFAddress) +} + +func TestVendorVFIOSelectUsesTiebreakAmongFreeVFs(t *testing.T) { + sysfs := newTestVendorVFIOSysfs(t) + sysfs.pickVFIndex = func(n int) int { return n - 1 } + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "0", testCreatableTypes) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.5", "43", "0", testCreatableTypes) + + device, err := sysfs.create(context.Background(), "NVIDIA L40S-1Q", "instance-1") + require.NoError(t, err) + assert.Equal(t, "0000:82:00.5", device.VFAddress) +} + +func TestVendorVFIOListProfilesExcludesQuarantinedFromAvailability(t *testing.T) { + resetVFHealthStore(t) + quarantineVF(t, "0000:82:00.4") + + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "0", testCreatableTypes) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.5", "43", "0", testCreatableTypes) + + vfs, err := sysfs.discoverVFs() + require.NoError(t, err) + profiles, err := sysfs.listProfiles(vfs) + require.NoError(t, err) + assert.Equal(t, 1, profileAvailability(profiles, "NVIDIA L40S-1Q")) +} diff --git a/lib/devices/vf_health.go b/lib/devices/vf_health.go new file mode 100644 index 000000000..b163db30d --- /dev/null +++ b/lib/devices/vf_health.go @@ -0,0 +1,436 @@ +package devices + +import ( + "encoding/json" + "fmt" + "log/slog" + "os" + "path/filepath" + "regexp" + "sort" + "sync" + "time" +) + +const ( + vfHealthFileVersion = 1 + defaultVFQuarantineThreshold = 2 +) + +type vfInitFailure struct { + InstanceID string `json:"instance_id,omitempty"` + AssignedAt string `json:"assigned_at,omitempty"` + ReportedAt time.Time `json:"reported_at"` +} + +type vfHealthRecord struct { + VFAddress string `json:"vf_address"` + Failures []vfInitFailure `json:"failures,omitempty"` + QuarantinedAt *time.Time `json:"quarantined_at,omitempty"` +} + +type vfHealthFile struct { + Version int `json:"version"` + Records []vfHealthRecord `json:"records"` +} + +// VFInitFailureReport describes one guest-reported driver init failure. +type VFInitFailureReport struct { + VFAddress string + InstanceID string + AssignedAt string +} + +// VFInitSuccessReport identifies the assignment that successfully initialized. +type VFInitSuccessReport struct { + VFAddress string + InstanceID string + AssignedAt string +} + +// VFReportOutcome describes how a failure report changed a VF's health state. +type VFReportOutcome int + +const ( + // VFReportUnchanged means the VF was already quarantined or this + // assignment was already recorded. + VFReportUnchanged VFReportOutcome = iota + // VFReportRecorded means the failure was tallied below the quarantine threshold. + VFReportRecorded + // VFReportQuarantined means this report crossed the threshold and quarantined the VF. + VFReportQuarantined +) + +// VFReportResult is the outcome of recording a driver init failure. +type VFReportResult struct { + Outcome VFReportOutcome + Failures int + Threshold int +} + +// VFSuccessResult describes how a successful init changed a VF's health state. +type VFSuccessResult struct { + Cleared int + Rescinded bool +} + +type vfHealthStore struct { + mu sync.Mutex + path string + records map[string]vfHealthRecord + threshold int + loadErr error +} + +var vfHealthAddressPattern = regexp.MustCompile(`^[0-9a-f]{4}:[0-9a-f]{2}:[0-9a-f]{2}\.[0-7]$`) + +var ( + vfHealth = &vfHealthStore{records: make(map[string]vfHealthRecord), threshold: defaultVFQuarantineThreshold} + vendorVFIOMu sync.Mutex +) + +func initVFHealth(path string) error { + vfHealth.mu.Lock() + defer vfHealth.mu.Unlock() + vfHealth.path = path + return vfHealth.loadLocked() +} + +// SetVFQuarantineThreshold configures the number of failed assignments +// required to quarantine a VF. +func SetVFQuarantineThreshold(n int) { + vfHealth.mu.Lock() + defer vfHealth.mu.Unlock() + vfHealth.threshold = n +} + +func (s *vfHealthStore) loadLocked() error { + s.records = make(map[string]vfHealthRecord) + s.loadErr = nil + + data, err := os.ReadFile(s.path) + if err != nil { + if os.IsNotExist(err) { + return nil + } + s.loadErr = fmt.Errorf("read VF health state: %w", err) + return s.loadErr + } + var state vfHealthFile + if err := json.Unmarshal(data, &state); err != nil { + s.loadErr = fmt.Errorf("unmarshal VF health state: %w", err) + return s.loadErr + } + if state.Version != vfHealthFileVersion { + s.loadErr = fmt.Errorf("validate VF health state: unsupported version %d", state.Version) + return s.loadErr + } + if state.Records == nil { + s.loadErr = fmt.Errorf("validate VF health state: expected a records array") + return s.loadErr + } + loaded := make(map[string]vfHealthRecord, len(state.Records)) + for i, record := range state.Records { + if !vfHealthAddressPattern.MatchString(record.VFAddress) { + s.loadErr = fmt.Errorf("validate VF health state record %d: invalid VF address %q", i, record.VFAddress) + return s.loadErr + } + if record.QuarantinedAt != nil && record.QuarantinedAt.IsZero() { + s.loadErr = fmt.Errorf("validate VF health state record %d: missing quarantine timestamp", i) + return s.loadErr + } + if record.QuarantinedAt == nil && len(record.Failures) == 0 { + s.loadErr = fmt.Errorf("validate VF health state record %d: neither quarantined nor any recorded failures", i) + return s.loadErr + } + assignments := make(map[string]struct{}, len(record.Failures)) + for j, failure := range record.Failures { + if failure.ReportedAt.IsZero() { + s.loadErr = fmt.Errorf("validate VF health state record %d failure %d: missing report timestamp", i, j) + return s.loadErr + } + key := failure.InstanceID + "\x00" + failure.AssignedAt + if _, exists := assignments[key]; exists { + s.loadErr = fmt.Errorf("validate VF health state record %d: duplicate failure for assignment %q", i, failure.InstanceID) + return s.loadErr + } + assignments[key] = struct{}{} + } + if _, exists := loaded[record.VFAddress]; exists { + s.loadErr = fmt.Errorf("validate VF health state record %d: duplicate VF address %q", i, record.VFAddress) + return s.loadErr + } + loaded[record.VFAddress] = record + } + s.records = loaded + return nil +} + +func (s *vfHealthStore) ensureLoadedLocked() error { + if s.loadErr == nil { + return nil + } + return s.loadLocked() +} + +func (s *vfHealthStore) checkedAddresses() (map[string]struct{}, error) { + s.mu.Lock() + defer s.mu.Unlock() + if err := s.ensureLoadedLocked(); err != nil { + return nil, fmt.Errorf("VF health state unavailable: %w", err) + } + addresses := make(map[string]struct{}, len(s.records)) + for address, record := range s.records { + if record.QuarantinedAt != nil { + addresses[address] = struct{}{} + } + } + return addresses, nil +} + +// VGPUAvailability returns free allocatable and quarantined VF counts. +func VGPUAvailability(framework VGPUFramework, vfs []VirtualFunction) (allocatable, quarantined int, err error) { + if framework != VGPUFrameworkVendorVFIO { + return countFreeVFs(vfs, nil), 0, nil + } + addresses, err := vfHealth.checkedAddresses() + if err != nil { + return 0, 0, err + } + for _, vf := range vfs { + if _, ok := addresses[vf.PCIAddress]; ok { + quarantined++ + } + } + return countFreeVFs(vfs, addresses), quarantined, nil +} + +func countFreeVFs(vfs []VirtualFunction, quarantined map[string]struct{}) int { + available := 0 + for _, vf := range vfs { + if vf.Allocated { + continue + } + if _, ok := quarantined[vf.PCIAddress]; !ok { + available++ + } + } + return available +} + +// ReportVFInitFailure records a guest-reported driver init failure and +// quarantines the VF once failures from enough distinct assignments accumulate. +func ReportVFInitFailure(report VFInitFailureReport) (VFReportResult, error) { + vendorVFIOMu.Lock() + defer vendorVFIOMu.Unlock() + return vfHealth.reportFailure(report) +} + +// ReportVFInitSuccess clears failures through a successful assignment. A +// quarantine is rescinded only when that assignment triggered it. +func ReportVFInitSuccess(report VFInitSuccessReport) (VFSuccessResult, error) { + vendorVFIOMu.Lock() + defer vendorVFIOMu.Unlock() + return vfHealth.reportSuccess(report) +} + +// VFHealthStoreUnavailable reports whether persisted state failed to load. +func VFHealthStoreUnavailable() bool { + vfHealth.mu.Lock() + defer vfHealth.mu.Unlock() + return vfHealth.loadErr != nil +} + +// TotalQuarantinedVFs returns the number of quarantined VFs in persisted state. +func TotalQuarantinedVFs() int { + vfHealth.mu.Lock() + defer vfHealth.mu.Unlock() + count := 0 + for _, record := range vfHealth.records { + if record.QuarantinedAt != nil { + count++ + } + } + return count +} + +func (s *vfHealthStore) sortedRecordsLocked() []vfHealthRecord { + records := make([]vfHealthRecord, 0, len(s.records)) + for _, record := range s.records { + records = append(records, record) + } + sort.Slice(records, func(i, j int) bool { return records[i].VFAddress < records[j].VFAddress }) + return records +} + +func (s *vfHealthStore) reportFailure(report VFInitFailureReport) (VFReportResult, error) { + s.mu.Lock() + defer s.mu.Unlock() + if err := s.ensureLoadedLocked(); err != nil { + return VFReportResult{}, err + } + if !vfHealthAddressPattern.MatchString(report.VFAddress) { + return VFReportResult{}, fmt.Errorf("invalid VF address %q", report.VFAddress) + } + + previous, existed := s.records[report.VFAddress] + result := VFReportResult{Failures: len(previous.Failures), Threshold: s.threshold} + if previous.QuarantinedAt != nil { + return result, nil + } + for _, failure := range previous.Failures { + if sameVFAssignment(failure, report.InstanceID, report.AssignedAt) { + return result, nil + } + } + + record := vfHealthRecord{ + VFAddress: report.VFAddress, + Failures: append(append([]vfInitFailure(nil), previous.Failures...), vfInitFailure{ + InstanceID: report.InstanceID, + AssignedAt: report.AssignedAt, + ReportedAt: time.Now().UTC(), + }), + } + result.Failures = len(record.Failures) + result.Outcome = VFReportRecorded + if result.Failures >= s.threshold { + now := time.Now().UTC() + record.QuarantinedAt = &now + result.Outcome = VFReportQuarantined + } + s.records[report.VFAddress] = record + if err := s.persistLocked(); err != nil { + if existed { + s.records[report.VFAddress] = previous + } else { + delete(s.records, report.VFAddress) + } + return VFReportResult{}, err + } + return result, nil +} + +func sameVFAssignment(failure vfInitFailure, instanceID, assignedAt string) bool { + return failure.InstanceID == instanceID && failure.AssignedAt == assignedAt +} + +func (s *vfHealthStore) reportSuccess(report VFInitSuccessReport) (VFSuccessResult, error) { + s.mu.Lock() + defer s.mu.Unlock() + if err := s.ensureLoadedLocked(); err != nil { + return VFSuccessResult{}, err + } + if !vfHealthAddressPattern.MatchString(report.VFAddress) { + return VFSuccessResult{}, fmt.Errorf("invalid VF address %q", report.VFAddress) + } + previous, ok := s.records[report.VFAddress] + if !ok || len(previous.Failures) == 0 { + return VFSuccessResult{}, nil + } + + match := -1 + for i, failure := range previous.Failures { + if sameVFAssignment(failure, report.InstanceID, report.AssignedAt) { + match = i + break + } + } + if previous.QuarantinedAt != nil && match != len(previous.Failures)-1 { + return VFSuccessResult{}, nil + } + + var remaining []vfInitFailure + if match >= 0 { + remaining = append([]vfInitFailure(nil), previous.Failures[match+1:]...) + } else { + remaining = failuresAfter(previous.Failures, report.AssignedAt) + } + result := VFSuccessResult{ + Cleared: len(previous.Failures) - len(remaining), + Rescinded: previous.QuarantinedAt != nil, + } + if result.Cleared == 0 { + return VFSuccessResult{}, nil + } + + if len(remaining) == 0 { + delete(s.records, report.VFAddress) + } else { + record := previous + record.Failures = remaining + s.records[report.VFAddress] = record + } + if err := s.persistLocked(); err != nil { + s.records[report.VFAddress] = previous + return VFSuccessResult{}, err + } + return result, nil +} + +// failuresAfter keeps any failure it cannot order against the success, so an +// unparseable timestamp never clears evidence. +func failuresAfter(failures []vfInitFailure, assignedAt string) []vfInitFailure { + successAt, err := time.Parse(time.RFC3339Nano, assignedAt) + if err != nil { + return failures + } + remaining := make([]vfInitFailure, 0, len(failures)) + for _, failure := range failures { + failureAt, err := time.Parse(time.RFC3339Nano, failure.AssignedAt) + if err != nil || failureAt.After(successAt) { + remaining = append(remaining, failure) + } + } + return remaining +} + +func (s *vfHealthStore) persistLocked() error { + if s.path == "" { + return nil + } + data, err := json.MarshalIndent(vfHealthFile{ + Version: vfHealthFileVersion, + Records: s.sortedRecordsLocked(), + }, "", " ") + if err != nil { + return fmt.Errorf("marshal VF health state: %w", err) + } + if err := os.MkdirAll(filepath.Dir(s.path), 0755); err != nil { + return fmt.Errorf("create VF health state dir: %w", err) + } + tmp := s.path + ".tmp" + f, err := os.OpenFile(tmp, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) + if err != nil { + return fmt.Errorf("create VF health state: %w", err) + } + if _, err := f.Write(data); err != nil { + f.Close() + os.Remove(tmp) + return fmt.Errorf("write VF health state: %w", err) + } + if err := f.Sync(); err != nil { + f.Close() + os.Remove(tmp) + return fmt.Errorf("sync VF health state: %w", err) + } + if err := f.Close(); err != nil { + os.Remove(tmp) + return fmt.Errorf("close VF health state: %w", err) + } + if err := os.Rename(tmp, s.path); err != nil { + os.Remove(tmp) + return fmt.Errorf("rename VF health state: %w", err) + } + dirPath := filepath.Dir(s.path) + dir, err := os.Open(dirPath) + if err != nil { + slog.Default().Warn("failed to open VF health state directory for sync", "path", dirPath, "error", err) + return nil + } + if err := dir.Sync(); err != nil { + slog.Default().Warn("failed to sync VF health state directory", "path", dirPath, "error", err) + } + _ = dir.Close() + return nil +} diff --git a/lib/devices/vf_health_test.go b/lib/devices/vf_health_test.go new file mode 100644 index 000000000..b8e009ea4 --- /dev/null +++ b/lib/devices/vf_health_test.go @@ -0,0 +1,436 @@ +package devices + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func resetVFHealthStore(t *testing.T) string { + t.Helper() + path := filepath.Join(t.TempDir(), "vf-health.json") + require.NoError(t, initVFHealth(path)) + t.Cleanup(func() { + vfHealth.mu.Lock() + defer vfHealth.mu.Unlock() + vfHealth.path = "" + vfHealth.records = make(map[string]vfHealthRecord) + vfHealth.threshold = defaultVFQuarantineThreshold + vfHealth.loadErr = nil + }) + return path +} + +func quarantinedVFs() []vfHealthRecord { + vfHealth.mu.Lock() + defer vfHealth.mu.Unlock() + records := vfHealth.sortedRecordsLocked() + result := records[:0] + for _, record := range records { + if record.QuarantinedAt != nil { + result = append(result, record) + } + } + return result +} + +func quarantineVF(t *testing.T, address string) { + t.Helper() + SetVFQuarantineThreshold(1) + result, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: address, InstanceID: "quarantine-helper"}) + require.NoError(t, err) + require.Equal(t, VFReportQuarantined, result.Outcome) + SetVFQuarantineThreshold(defaultVFQuarantineThreshold) +} + +func TestVGPUAvailability(t *testing.T) { + resetVFHealthStore(t) + quarantineVF(t, "0000:82:00.4") + vfs := []VirtualFunction{ + {PCIAddress: "0000:82:00.4"}, + {PCIAddress: "0000:82:00.5", Allocated: true}, + {PCIAddress: "0000:82:00.6"}, + } + + available, quarantined, err := VGPUAvailability(VGPUFrameworkVendorVFIO, vfs) + require.NoError(t, err) + assert.Equal(t, 1, available) + assert.Equal(t, 1, quarantined) + + available, quarantined, err = VGPUAvailability(VGPUFrameworkMdev, vfs) + require.NoError(t, err) + assert.Equal(t, 2, available) + assert.Zero(t, quarantined) +} + +func TestVGPUAvailabilityExcludesOnlyQuarantinedVFs(t *testing.T) { + resetVFHealthStore(t) + result, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:82:00.4", InstanceID: "instance-1"}) + require.NoError(t, err) + require.Equal(t, VFReportRecorded, result.Outcome) + + available, quarantined, err := VGPUAvailability(VGPUFrameworkVendorVFIO, []VirtualFunction{{PCIAddress: "0000:82:00.4"}}) + require.NoError(t, err) + assert.Equal(t, 1, available, "a below-threshold failure tally must not remove the VF from placement") + assert.Zero(t, quarantined) +} + +func TestVGPUAvailabilityFailsWhenStoreUnavailable(t *testing.T) { + path := resetVFHealthStore(t) + require.NoError(t, os.WriteFile(path, []byte("not json"), 0o644)) + require.Error(t, initVFHealth(path)) + + _, _, err := VGPUAvailability(VGPUFrameworkVendorVFIO, []VirtualFunction{{PCIAddress: "0000:82:00.4"}}) + require.ErrorContains(t, err, "VF health state unavailable") + + available, quarantined, err := VGPUAvailability(VGPUFrameworkMdev, []VirtualFunction{{PCIAddress: "0000:82:00.4"}}) + require.NoError(t, err) + assert.Equal(t, 1, available) + assert.Zero(t, quarantined) +} + +func TestReportVFInitFailureQuarantinesAtThreshold(t *testing.T) { + path := resetVFHealthStore(t) + + result, err := ReportVFInitFailure(VFInitFailureReport{ + VFAddress: "0000:e3:00.4", + InstanceID: "instance-1", + AssignedAt: "2026-08-20T15:00:00Z", + }) + require.NoError(t, err) + assert.Equal(t, VFReportRecorded, result.Outcome) + assert.Equal(t, 1, result.Failures) + assert.Equal(t, defaultVFQuarantineThreshold, result.Threshold) + assert.Empty(t, quarantinedVFs(), "one failure must not quarantine at the default threshold") + + result, err = ReportVFInitFailure(VFInitFailureReport{ + VFAddress: "0000:e3:00.4", + InstanceID: "instance-2", + AssignedAt: "2026-08-20T16:00:00Z", + }) + require.NoError(t, err) + assert.Equal(t, VFReportQuarantined, result.Outcome) + assert.Equal(t, 2, result.Failures) + + records := quarantinedVFs() + require.Len(t, records, 1) + assert.Equal(t, 1, TotalQuarantinedVFs()) + assert.Equal(t, "0000:e3:00.4", records[0].VFAddress) + require.NotNil(t, records[0].QuarantinedAt) + require.Len(t, records[0].Failures, 2) + assert.Equal(t, "instance-1", records[0].Failures[0].InstanceID) + + result, err = ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:e3:00.4", InstanceID: "instance-3"}) + require.NoError(t, err) + assert.Equal(t, VFReportUnchanged, result.Outcome) + + require.NoError(t, initVFHealth(path)) + reloaded := quarantinedVFs() + require.Len(t, reloaded, 1) + assert.Equal(t, "0000:e3:00.4", reloaded[0].VFAddress) + require.Len(t, reloaded[0].Failures, 2) +} + +func TestReportVFInitFailureDeduplicatesAssignments(t *testing.T) { + resetVFHealthStore(t) + + report := VFInitFailureReport{ + VFAddress: "0000:e3:00.4", + InstanceID: "instance-1", + AssignedAt: "2026-08-20T15:00:00Z", + } + result, err := ReportVFInitFailure(report) + require.NoError(t, err) + assert.Equal(t, VFReportRecorded, result.Outcome) + + result, err = ReportVFInitFailure(report) + require.NoError(t, err) + assert.Equal(t, VFReportUnchanged, result.Outcome) + assert.Equal(t, 1, result.Failures) + assert.Empty(t, quarantinedVFs(), "a rescanned assignment must not count toward the threshold twice") +} + +func TestReportVFInitFailureRespectsConfiguredThreshold(t *testing.T) { + resetVFHealthStore(t) + SetVFQuarantineThreshold(3) + + for i, instance := range []string{"instance-1", "instance-2"} { + result, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:e3:00.4", InstanceID: instance}) + require.NoError(t, err) + assert.Equal(t, VFReportRecorded, result.Outcome) + assert.Equal(t, i+1, result.Failures) + assert.Equal(t, 3, result.Threshold) + } + + result, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:e3:00.4", InstanceID: "instance-3"}) + require.NoError(t, err) + assert.Equal(t, VFReportQuarantined, result.Outcome) +} + +func TestReportVFInitSuccessClearsFailureTally(t *testing.T) { + path := resetVFHealthStore(t) + report := VFInitFailureReport{ + VFAddress: "0000:e3:00.4", + InstanceID: "instance-1", + AssignedAt: "2026-08-20T15:00:00Z", + } + + _, err := ReportVFInitFailure(report) + require.NoError(t, err) + + success := VFInitSuccessReport{ + VFAddress: report.VFAddress, + InstanceID: "instance-2", + AssignedAt: "2026-08-20T16:00:00Z", + } + successResult, err := ReportVFInitSuccess(success) + require.NoError(t, err) + assert.Equal(t, 1, successResult.Cleared) + assert.False(t, successResult.Rescinded) + + successResult, err = ReportVFInitSuccess(success) + require.NoError(t, err) + assert.Zero(t, successResult.Cleared) + + require.NoError(t, initVFHealth(path)) + result, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: report.VFAddress, InstanceID: "instance-3"}) + require.NoError(t, err) + assert.Equal(t, VFReportRecorded, result.Outcome) + assert.Equal(t, 1, result.Failures) +} + +func TestReportVFInitSuccessRescindsQuarantineTriggeredByAssignment(t *testing.T) { + resetVFHealthStore(t) + vf := "0000:e3:00.4" + _, err := ReportVFInitFailure(VFInitFailureReport{ + VFAddress: vf, + InstanceID: "instance-1", + AssignedAt: "2026-08-20T14:00:00Z", + }) + require.NoError(t, err) + trigger := VFInitFailureReport{ + VFAddress: vf, + InstanceID: "instance-2", + AssignedAt: "2026-08-20T15:00:00Z", + } + result, err := ReportVFInitFailure(trigger) + require.NoError(t, err) + require.Equal(t, VFReportQuarantined, result.Outcome) + + success, err := ReportVFInitSuccess(VFInitSuccessReport{ + VFAddress: trigger.VFAddress, + InstanceID: trigger.InstanceID, + AssignedAt: trigger.AssignedAt, + }) + require.NoError(t, err) + assert.Equal(t, 2, success.Cleared) + assert.True(t, success.Rescinded) + assert.Empty(t, quarantinedVFs()) +} + +func TestReportVFInitSuccessPreservesNewerFailures(t *testing.T) { + resetVFHealthStore(t) + SetVFQuarantineThreshold(3) + vf := "0000:e3:00.4" + for _, report := range []VFInitFailureReport{ + {VFAddress: vf, InstanceID: "instance-1", AssignedAt: "2026-08-20T15:00:00Z"}, + {VFAddress: vf, InstanceID: "instance-2", AssignedAt: "2026-08-20T16:00:00Z"}, + } { + _, err := ReportVFInitFailure(report) + require.NoError(t, err) + } + + success, err := ReportVFInitSuccess(VFInitSuccessReport{ + VFAddress: vf, + InstanceID: "instance-1", + AssignedAt: "2026-08-20T15:00:00Z", + }) + require.NoError(t, err) + assert.Equal(t, 1, success.Cleared) + + result, err := ReportVFInitFailure(VFInitFailureReport{ + VFAddress: vf, + InstanceID: "instance-2", + AssignedAt: "2026-08-20T16:00:00Z", + }) + require.NoError(t, err) + assert.Equal(t, VFReportUnchanged, result.Outcome) + assert.Equal(t, 1, result.Failures) +} + +func TestReportVFInitSuccessNeverClearsAnotherAssignmentsQuarantine(t *testing.T) { + resetVFHealthStore(t) + quarantineVF(t, "0000:e3:00.4") + + result, err := ReportVFInitSuccess(VFInitSuccessReport{ + VFAddress: "0000:e3:00.4", + InstanceID: "another-instance", + }) + require.NoError(t, err) + assert.Zero(t, result.Cleared) + assert.False(t, result.Rescinded) + require.Len(t, quarantinedVFs(), 1) +} + +func TestReportVFInitSuccessWithUnparseableAssignedAtClearsNothing(t *testing.T) { + resetVFHealthStore(t) + _, err := ReportVFInitFailure(VFInitFailureReport{ + VFAddress: "0000:e3:00.4", + InstanceID: "instance-1", + AssignedAt: "2026-08-20T15:00:00Z", + }) + require.NoError(t, err) + + result, err := ReportVFInitSuccess(VFInitSuccessReport{ + VFAddress: "0000:e3:00.4", + InstanceID: "instance-2", + }) + require.NoError(t, err) + assert.Zero(t, result.Cleared, "a success that cannot be ordered against the tally must clear nothing") +} + +func TestReportVFInitFailureRejectsInvalidAddress(t *testing.T) { + resetVFHealthStore(t) + + _, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: "not-a-pci-address"}) + require.ErrorContains(t, err, "invalid VF address") + _, err = ReportVFInitSuccess(VFInitSuccessReport{VFAddress: "not-a-pci-address"}) + require.ErrorContains(t, err, "invalid VF address") + assert.Empty(t, quarantinedVFs()) +} + +func TestReportVFInitFailureRollsBackOnPersistFailure(t *testing.T) { + resetVFHealthStore(t) + blocker := filepath.Join(t.TempDir(), "blocker") + require.NoError(t, os.WriteFile(blocker, nil, 0644)) + vfHealth.path = filepath.Join(blocker, "vf-health.json") + + _, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:e3:00.4", InstanceID: "instance-1"}) + require.Error(t, err) + + vfHealth.mu.Lock() + _, exists := vfHealth.records["0000:e3:00.4"] + vfHealth.mu.Unlock() + assert.False(t, exists, "a failure whose persist failed must be retried by the next report") +} + +func TestReportVFInitSuccessRollsBackOnPersistFailure(t *testing.T) { + resetVFHealthStore(t) + _, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:e3:00.4", InstanceID: "instance-1"}) + require.NoError(t, err) + + blocker := filepath.Join(t.TempDir(), "blocker") + require.NoError(t, os.WriteFile(blocker, nil, 0644)) + goodPath := vfHealth.path + vfHealth.path = filepath.Join(blocker, "vf-health.json") + + _, err = ReportVFInitSuccess(VFInitSuccessReport{VFAddress: "0000:e3:00.4", InstanceID: "instance-1"}) + require.Error(t, err) + vfHealth.path = goodPath + + vfHealth.mu.Lock() + record, exists := vfHealth.records["0000:e3:00.4"] + vfHealth.mu.Unlock() + require.True(t, exists, "a clear whose persist failed must be restored in memory") + assert.Len(t, record.Failures, 1) +} + +func TestCheckedAddressesFailsClosedOnUnloadedState(t *testing.T) { + path := resetVFHealthStore(t) + quarantineVF(t, "0000:e3:00.4") + + require.NoError(t, os.WriteFile(path, []byte("not json"), 0644)) + require.Error(t, initVFHealth(path)) + + _, err := vfHealth.checkedAddresses() + require.Error(t, err) + + restored := `{"version":1,"records":[{"vf_address":"0000:e3:00.4","quarantined_at":"2026-08-20T00:00:00Z"}]}` + require.NoError(t, os.WriteFile(path, []byte(restored), 0644)) + addresses, err := vfHealth.checkedAddresses() + require.NoError(t, err) + assert.Contains(t, addresses, "0000:e3:00.4") +} + +func TestCheckedAddressesFailsClosedOnInvalidRecord(t *testing.T) { + tests := []struct { + name string + state string + wantErr string + }{ + { + name: "unsupported version", + state: `{"version":2,"records":[]}`, + wantErr: "unsupported version 2", + }, + { + name: "missing records", + state: `{"version":1}`, + wantErr: "expected a records array", + }, + { + name: "invalid address", + state: `{"version":1,"records":[{"vf_address":"not-a-pci-address","quarantined_at":"2026-08-20T00:00:00Z"}]}`, + wantErr: "invalid VF address", + }, + { + name: "neither quarantined nor failed", + state: `{"version":1,"records":[{"vf_address":"0000:e3:00.4"}]}`, + wantErr: "neither quarantined nor any recorded failures", + }, + { + name: "failure missing report timestamp", + state: `{"version":1,"records":[{"vf_address":"0000:e3:00.4","failures":[{"instance_id":"instance-1"}]}]}`, + wantErr: "missing report timestamp", + }, + { + name: "duplicate assignment", + state: `{"version":1,"records":[{"vf_address":"0000:e3:00.4","failures":[{"instance_id":"instance-1","assigned_at":"a","reported_at":"2026-08-20T00:00:00Z"},{"instance_id":"instance-1","assigned_at":"a","reported_at":"2026-08-21T00:00:00Z"}]}]}`, + wantErr: "duplicate failure for assignment", + }, + { + name: "duplicate address", + state: `{"version":1,"records":[{"vf_address":"0000:e3:00.4","quarantined_at":"2026-08-20T00:00:00Z"},{"vf_address":"0000:e3:00.4","quarantined_at":"2026-08-21T00:00:00Z"}]}`, + wantErr: "duplicate VF address", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + path := resetVFHealthStore(t) + require.NoError(t, os.WriteFile(path, []byte(tt.state), 0644)) + require.ErrorContains(t, initVFHealth(path), tt.wantErr) + assert.True(t, VFHealthStoreUnavailable()) + assert.Empty(t, quarantinedVFs()) + + _, err := vfHealth.checkedAddresses() + require.Error(t, err) + }) + } +} + +func TestReportVFInitFailureRefusesToClobberUnloadedState(t *testing.T) { + path := resetVFHealthStore(t) + quarantineVF(t, "0000:e3:00.4") + + require.NoError(t, os.WriteFile(path, []byte("not json"), 0644)) + require.Error(t, initVFHealth(path)) + + _, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:e3:00.5"}) + require.Error(t, err) + _, err = ReportVFInitSuccess(VFInitSuccessReport{VFAddress: "0000:e3:00.5"}) + require.Error(t, err) + data, err := os.ReadFile(path) + require.NoError(t, err) + assert.Equal(t, "not json", string(data), "a failed load must not be overwritten by later reports") + + restored := `{"version":1,"records":[{"vf_address":"0000:e3:00.4","quarantined_at":"2026-08-20T00:00:00Z"}]}` + require.NoError(t, os.WriteFile(path, []byte(restored), 0644)) + quarantineVF(t, "0000:e3:00.5") + records := quarantinedVFs() + require.Len(t, records, 2, "reload must recover the previously persisted quarantine") + assert.Equal(t, "0000:e3:00.4", records[0].VFAddress) +} diff --git a/lib/instances/create.go b/lib/instances/create.go index 525f2c262..4b0f97604 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -272,12 +272,6 @@ func (m *manager) createInstance( // whatever devices have been attached when cleanup runs. var attachedDeviceIDs []string var resolvedDeviceIDs []string - var gpuDevice *devices.VGPUDevice - var gpuProfile string - var gpuFramework devices.VGPUFramework - var gpuDevicePath string - var gpuMdevUUID string - var gpuAssignedAt *time.Time retention := vgpuRetention{instanceID: id} defer retention.deferWrapPending(&retErr) @@ -297,53 +291,6 @@ func (m *manager) createInstance( }) } - // Handle vGPU profile request - if req.GPU != nil && req.GPU.Profile != "" { - retentionStub := func() StoredMetadata { - return StoredMetadata{ - Id: id, - Name: req.Name, - Image: req.Image, - ResolvedImage: resolvedImageRef, - Platform: imageInfo.Platform, - CreatedAt: m.nowUTC(), - HypervisorType: hvType, - HypervisorVersion: hvVersion, - SocketPath: m.paths.InstanceSocket(id, starter.SocketName()), - DataDir: m.paths.InstanceDir(id), - } - } - log.InfoContext(ctx, "creating vGPU", "instance_id", id, "profile", req.GPU.Profile) - gpuDevice, err = m.createVGPUDevice(ctx, req.GPU.Profile, id) - if err != nil { - retention.retainFromCreateError(retentionStub(), m.nowUTC(), err) - log.ErrorContext(ctx, "failed to create vGPU", "profile", req.GPU.Profile, "error", err) - return nil, wrapCreateVGPUErr(req.GPU.Profile, err) - } - gpuProfile = gpuDevice.ProfileName - gpuFramework = gpuDevice.Framework - gpuDevicePath = gpuDevice.SysfsPath - gpuMdevUUID = gpuDevice.MdevUUID - assignedAt := m.nowUTC() - gpuAssignedAt = &assignedAt - log.InfoContext(ctx, "created vGPU", "instance_id", id, "profile", gpuProfile, "uuid", gpuMdevUUID) - - // Add vGPU cleanup to stack - cu.Add(func() { - log.DebugContext(ctx, "destroying vGPU on cleanup", "instance_id", id, "uuid", gpuDevice.MdevUUID) - assignment := devices.VGPUAssignment{ - Framework: gpuDevice.Framework, - DevicePath: gpuDevice.SysfsPath, - MdevUUID: gpuDevice.MdevUUID, - InstanceID: id, - } - if err := m.destroyVGPUAssignment(ctx, assignment); err != nil { - log.WarnContext(ctx, "failed to destroy vGPU on cleanup", "instance_id", id, "uuid", gpuDevice.MdevUUID, "error", err) - retention.retainFromDevice(retentionStub(), gpuDevice, *gpuAssignedAt) - } - }) - } - if len(req.Devices) > 0 && m.deviceManager != nil { for _, deviceRef := range req.Devices { device, err := m.deviceManager.GetDevice(ctx, deviceRef) @@ -412,11 +359,6 @@ func (m *manager) createInstance( VsockCID: vsockCID, VsockSocket: vsockSocket, Devices: resolvedDeviceIDs, - GPUProfile: gpuProfile, - GPUFramework: gpuFramework, - GPUDevicePath: gpuDevicePath, - GPUMdevUUID: gpuMdevUUID, - GPUAssignedAt: gpuAssignedAt, Entrypoint: req.Entrypoint, Cmd: req.Cmd, SkipKernelHeaders: req.SkipKernelHeaders, @@ -559,6 +501,36 @@ func (m *manager) createInstance( stored.StartedAt = &bootStart stored.Phases.Record(phasetracking.PhaseCreated, bootStart) + // Allocate the vGPU after disk, network, volume, and config setup so the + // assignment can be persisted immediately. + if req.GPU != nil && req.GPU.Profile != "" { + log.InfoContext(ctx, "creating vGPU", "instance_id", id, "profile", req.GPU.Profile) + gpuDevice, err := m.createVGPUDevice(ctx, req.GPU.Profile, id) + if err != nil { + retention.retainFromCreateError(*stored, m.nowUTC(), err) + log.ErrorContext(ctx, "failed to create vGPU", "profile", req.GPU.Profile, "error", err) + return nil, wrapCreateVGPUErr(req.GPU.Profile, err) + } + assignedAt := m.nowUTC() + stored.GPUProfile = gpuDevice.ProfileName + setStoredVGPUDevice(stored, gpuDevice, assignedAt) + log.InfoContext(ctx, "created vGPU", "instance_id", id, "profile", stored.GPUProfile, "uuid", stored.GPUMdevUUID) + + cu.Add(func() { + log.DebugContext(ctx, "destroying vGPU on cleanup", "instance_id", id, "uuid", gpuDevice.MdevUUID) + assignment := devices.VGPUAssignment{ + Framework: gpuDevice.Framework, + DevicePath: gpuDevice.SysfsPath, + MdevUUID: gpuDevice.MdevUUID, + InstanceID: id, + } + if err := m.destroyVGPUAssignment(ctx, assignment); err != nil { + log.WarnContext(ctx, "failed to destroy vGPU on cleanup", "instance_id", id, "uuid", gpuDevice.MdevUUID, "error", err) + retention.retainFromDevice(*stored, gpuDevice, assignedAt) + } + }) + } + // 18. Save metadata log.DebugContext(ctx, "saving instance metadata", "instance_id", id) meta := &metadata{StoredMetadata: *stored} diff --git a/lib/instances/fork_test.go b/lib/instances/fork_test.go index c0bed670e..24d7903a8 100644 --- a/lib/instances/fork_test.go +++ b/lib/instances/fork_test.go @@ -83,7 +83,7 @@ func TestForkInstanceRejectsVGPURetentionRecord(t *testing.T) { _, err = manager.ForkInstance(ctx, sourceID, ForkInstanceRequest{Name: "fork-vgpu-retention-copy"}) require.ErrorIs(t, err, ErrInvalidState) - require.ErrorContains(t, err, "delete it to release the assignment") + require.ErrorContains(t, err, "delete-only record of a failed create") } func TestForkInstance_VZStoppedSourceSupported(t *testing.T) { diff --git a/lib/instances/lifecycle_noop_test.go b/lib/instances/lifecycle_noop_test.go index 77d15d733..8a0b4ca66 100644 --- a/lib/instances/lifecycle_noop_test.go +++ b/lib/instances/lifecycle_noop_test.go @@ -315,7 +315,7 @@ func TestStartRejectsVGPURetentionRecord(t *testing.T) { _, err = m.StartInstance(context.Background(), id, StartInstanceRequest{}) require.ErrorIs(t, err, ErrInvalidState) - require.ErrorContains(t, err, "delete it to release the assignment") + require.ErrorContains(t, err, "delete-only record of a failed create") stored, err := m.loadMetadata(id) require.NoError(t, err) diff --git a/lib/instances/logs.go b/lib/instances/logs.go index f10e8537d..85cd11427 100644 --- a/lib/instances/logs.go +++ b/lib/instances/logs.go @@ -35,6 +35,8 @@ var ErrLogNotFound = fmt.Errorf("log file not found") var appLogNoiseMarkers = []string{ "HYPEMAN-PROGRAM-START", "HYPEMAN-AGENT-READY", + "HYPEMAN-GPU-INIT-FAILED", + "HYPEMAN-GPU-INIT-OK", "HYPEMAN-HEADERS-START", "HYPEMAN-HEADERS-READY", "HYPEMAN-HEADERS-FAILED", diff --git a/lib/instances/logs_test.go b/lib/instances/logs_test.go index 3ce9ac80a..72d531a25 100644 --- a/lib/instances/logs_test.go +++ b/lib/instances/logs_test.go @@ -24,6 +24,11 @@ func TestShouldSkipAppLogLine(t *testing.T) { line: "2026-03-10T10:00:01Z [INFO] HYPEMAN-AGENT-READY ts=2026-03-10T10:00:01Z", want: true, }, + { + name: "gpu init marker", + line: "2026/08/20 15:04:05 [guest-agent] HYPEMAN-GPU-INIT-OK ts=2026-08-20T15:04:05Z", + want: true, + }, { name: "headers marker", line: "2026-03-10T10:00:02Z [INFO] HYPEMAN-HEADERS-READY", diff --git a/lib/instances/metrics.go b/lib/instances/metrics.go index e14deb49b..e83458680 100644 --- a/lib/instances/metrics.go +++ b/lib/instances/metrics.go @@ -111,6 +111,7 @@ type Metrics struct { vgpuReconcileFailuresTotal metric.Int64Counter vgpuStaleReleaseFailuresTotal metric.Int64Counter vgpuAssignmentsRetainedTotal metric.Int64Counter + vgpuReconcileLivenessUncertainTotal metric.Int64Counter tracer trace.Tracer } @@ -311,6 +312,14 @@ func newInstanceMetrics(meter metric.Meter, tracer trace.Tracer, m *manager) (*M return nil, err } + vgpuReconcileLivenessUncertainTotal, err := meter.Int64Counter( + "hypeman_instances_vgpu_reconcile_liveness_uncertain_total", + metric.WithDescription("Total vGPU reconcile checks that preserved an assignment because hypervisor liveness was uncertain"), + ) + if err != nil { + return nil, err + } + // Register observable gauge for instance counts by state instancesTotal, err := meter.Int64ObservableGauge( "hypeman_instances_total", @@ -508,6 +517,7 @@ func newInstanceMetrics(meter metric.Meter, tracer trace.Tracer, m *manager) (*M vgpuReconcileFailuresTotal: vgpuReconcileFailuresTotal, vgpuStaleReleaseFailuresTotal: vgpuStaleReleaseFailuresTotal, vgpuAssignmentsRetainedTotal: vgpuAssignmentsRetainedTotal, + vgpuReconcileLivenessUncertainTotal: vgpuReconcileLivenessUncertainTotal, tracer: tracer, }, nil } @@ -633,6 +643,13 @@ func (m *manager) recordVGPURetainedAssignment(ctx context.Context, operation vg )) } +func (m *manager) recordVGPUReconcileLivenessUncertain(ctx context.Context) { + if m.metrics == nil { + return + } + m.metrics.vgpuReconcileLivenessUncertainTotal.Add(ctx, 1) +} + // recordStateTransition records a state transition with hypervisor label. func (m *manager) recordStateTransition(ctx context.Context, fromState, toState string, hvType hypervisor.Type) { if m.metrics == nil { diff --git a/lib/instances/metrics_test.go b/lib/instances/metrics_test.go index 525002cbb..32944e79b 100644 --- a/lib/instances/metrics_test.go +++ b/lib/instances/metrics_test.go @@ -161,6 +161,27 @@ func TestSnapshotCompressionMetrics_RecordAndObserve(t *testing.T) { assert.Equal(t, "skipped", metricLabel(t, waitDurations.DataPoints[0].Attributes, "outcome")) } +func TestVGPUReconcileLivenessUncertainMetric(t *testing.T) { + t.Parallel() + + reader := otelmetric.NewManualReader() + provider := otelmetric.NewMeterProvider(otelmetric.WithReader(reader)) + m := &manager{paths: paths.New(t.TempDir())} + metrics, err := newInstanceMetrics(provider.Meter("test"), nil, m) + require.NoError(t, err) + m.metrics = metrics + + m.recordVGPUReconcileLivenessUncertain(t.Context()) + + var rm metricdata.ResourceMetrics + require.NoError(t, reader.Collect(t.Context(), &rm)) + metricValue := findMetric(t, rm, "hypeman_instances_vgpu_reconcile_liveness_uncertain_total") + uncertain, ok := metricValue.Data.(metricdata.Sum[int64]) + require.True(t, ok) + require.Len(t, uncertain.DataPoints, 1) + assert.Equal(t, int64(1), uncertain.DataPoints[0].Value) +} + func TestLifecycleEventMetrics_ObserveSubscribersQueueDepthAndDrops(t *testing.T) { t.Parallel() diff --git a/lib/instances/process_identity.go b/lib/instances/process_identity.go index 7e8500402..48891092a 100644 --- a/lib/instances/process_identity.go +++ b/lib/instances/process_identity.go @@ -202,9 +202,10 @@ func classifyResolvedHypervisorOwner(socketPath string, stored, resolved int, er } // Ambiguous ownership is treated as live; this must not authorize teardown. -func hypervisorMayBeAlive(id HypervisorProcessIdentity, socketPath string) bool { +// The error is returned so callers can surface why teardown was suppressed. +func hypervisorMayBeAlive(id HypervisorProcessIdentity, socketPath string) (bool, error) { pid, err := resolveLiveHypervisorPID(id, socketPath) - return err != nil || pid > 0 + return err != nil || pid > 0, err } // ProcessExists reports whether pid belongs to a live, non-zombie process. diff --git a/lib/instances/process_identity_linux_test.go b/lib/instances/process_identity_linux_test.go index 2ec201450..0ec559152 100644 --- a/lib/instances/process_identity_linux_test.go +++ b/lib/instances/process_identity_linux_test.go @@ -164,6 +164,10 @@ func TestResolveLiveHypervisorPIDFailsClosedWithoutSocketOrIdentity(t *testing.T resolved, err := resolveLiveHypervisorPID(HypervisorProcessIdentity{HypervisorPID: &pid}, "") require.ErrorContains(t, err, "without a socket path") assert.Zero(t, resolved) + + mayBeAlive, err := hypervisorMayBeAlive(HypervisorProcessIdentity{HypervisorPID: &pid}, "") + require.ErrorContains(t, err, "without a socket path") + assert.True(t, mayBeAlive, "ambiguous ownership must still fail closed") } func TestSocketListenerHelper(t *testing.T) { diff --git a/lib/instances/snapshot_test.go b/lib/instances/snapshot_test.go index 31f0a4343..e766515e7 100644 --- a/lib/instances/snapshot_test.go +++ b/lib/instances/snapshot_test.go @@ -72,7 +72,7 @@ func TestCreateSnapshotRejectsVGPURetentionRecord(t *testing.T) { Name: "snapshot-vgpu-retention", }) require.ErrorIs(t, err, ErrInvalidState) - require.ErrorContains(t, err, "delete it to release the assignment") + require.ErrorContains(t, err, "delete-only record of a failed create") } func TestRestoreSnapshotRejectsVGPURetentionRecord(t *testing.T) { @@ -101,7 +101,7 @@ func TestRestoreSnapshotRejectsVGPURetentionRecord(t *testing.T) { TargetHypervisor: mgr.defaultHypervisor, }) require.ErrorIs(t, err, ErrInvalidState) - require.ErrorContains(t, err, "delete it to release the assignment") + require.ErrorContains(t, err, "delete-only record of a failed create") stored, err := mgr.loadMetadata(sourceID) require.NoError(t, err) diff --git a/lib/instances/start.go b/lib/instances/start.go index ea8c2a6f4..e2893dbc7 100644 --- a/lib/instances/start.go +++ b/lib/instances/start.go @@ -170,6 +170,15 @@ func (m *manager) startInstance( } } + // Archive before assigning a VF so stale reports cannot count against the new assignment. + if err := m.archiveAppLogForBoot(id); err != nil { + if stored.GPUProfile != "" { + log.ErrorContext(ctx, "failed to archive app log before start", "instance_id", id, "error", err) + return nil, fmt.Errorf("archive app log before start: %w", err) + } + log.WarnContext(ctx, "failed to archive app log before start", "instance_id", id, "error", err) + } + // 4b. Recreate the vGPU if this instance had a GPU profile // Note: GPU availability was already validated in step 2b if stored.GPUProfile != "" { @@ -224,10 +233,6 @@ func (m *manager) startInstance( } configDiskSpanEnd(nil) - if err := m.archiveAppLogForBoot(id); err != nil { - log.WarnContext(ctx, "failed to archive app log before start", "instance_id", id, "error", err) - } - // 6. Start hypervisor and boot VM (reuses logic from create) bootStart := time.Now().UTC() stored.StartedAt = &bootStart diff --git a/lib/instances/storage.go b/lib/instances/storage.go index 1a4d325b0..f2112162d 100644 --- a/lib/instances/storage.go +++ b/lib/instances/storage.go @@ -188,27 +188,34 @@ func removeAllWithRetry(path string, removeAll func(string) error, sleep func(ti } func (m *manager) listMetadataFiles() ([]string, error) { - return m.walkMetadataFiles(false) + files, _, err := m.walkMetadataFiles() + return files, err } +// listMetadataFilesStrict returns readable metadata paths and joins any stat +// errors other than absence. Fail-closed callers (the vGPU release claim scan +// and startup reconcile protection) use it so an unreadable instance is an +// error instead of silently missing. func (m *manager) listMetadataFilesStrict() ([]string, error) { - return m.walkMetadataFiles(true) + files, statErr, err := m.walkMetadataFiles() + return files, errors.Join(statErr, err) } -func (m *manager) walkMetadataFiles(failOnStatError bool) ([]string, error) { +func (m *manager) walkMetadataFiles() (files []string, statErr error, err error) { guestsDir := m.paths.GuestsDir() // Ensure guests directory exists if err := os.MkdirAll(guestsDir, 0755); err != nil { - return nil, fmt.Errorf("create guests directory: %w", err) + return nil, nil, fmt.Errorf("create guests directory: %w", err) } entries, err := os.ReadDir(guestsDir) if err != nil { - return nil, fmt.Errorf("read guests directory: %w", err) + return nil, nil, fmt.Errorf("read guests directory: %w", err) } - var metaFiles []string + metaFiles := make([]string, 0, len(entries)) + var statErrs []error for _, entry := range entries { if !entry.IsDir() { continue @@ -217,10 +224,10 @@ func (m *manager) walkMetadataFiles(failOnStatError bool) ([]string, error) { metaPath := filepath.Join(guestsDir, entry.Name(), "metadata.json") if _, err := os.Stat(metaPath); err == nil { metaFiles = append(metaFiles, metaPath) - } else if failOnStatError && !os.IsNotExist(err) { - return nil, fmt.Errorf("stat metadata for instance %s: %w", entry.Name(), err) + } else if !os.IsNotExist(err) { + statErrs = append(statErrs, fmt.Errorf("stat metadata for instance %s: %w", entry.Name(), err)) } } - return metaFiles, nil + return metaFiles, errors.Join(statErrs...), nil } diff --git a/lib/instances/types.go b/lib/instances/types.go index ed32f5b5d..15b91bf87 100644 --- a/lib/instances/types.go +++ b/lib/instances/types.go @@ -156,7 +156,7 @@ type StoredMetadata struct { GPUDevicePath string GPUMdevUUID string // populated for mdev-backed vGPUs GPUAssignedAt *time.Time // set before hypervisor startup to bound crash recovery protection - GPURetainedForCleanup bool // delete-only stub holding a vGPU assignment + GPURetainedForCleanup bool // delete-only failed-create record; the assignment may already be released // Command overrides (like docker run ) Entrypoint []string // Override image entrypoint (nil = use image default) diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index 7b70e807c..097d80680 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -27,7 +27,7 @@ func (e *VGPUCleanupPendingError) Error() string { func (e *VGPUCleanupPendingError) Unwrap() error { return e.Err } -var errVGPURetentionStub = fmt.Errorf("%w: instance retains a vGPU assignment from a failed create and has no boot configuration; delete it to release the assignment", ErrInvalidState) +var errVGPURetentionStub = fmt.Errorf("%w: instance is a delete-only record of a failed create and has no boot configuration; delete it to remove the record and release any retained vGPU assignment", ErrInvalidState) func (m *manager) createVGPUDevice(ctx context.Context, profileName, instanceID string) (*devices.VGPUDevice, error) { create := m.createVGPU diff --git a/lib/instances/vgpu_reconcile.go b/lib/instances/vgpu_reconcile.go index 066c9bdee..a5cb0220d 100644 --- a/lib/instances/vgpu_reconcile.go +++ b/lib/instances/vgpu_reconcile.go @@ -85,7 +85,11 @@ func (m *manager) reconcileVGPUAssignments(ctx context.Context) (map[string]stru if storedVGPUDevicePath(stored) == "" { continue } - hypervisorLive := hypervisorMayBeAlive(stored.HypervisorProcessIdentity, stored.SocketPath) + // The socket-ownership check runs even without a persisted PID: a VMM + // whose post-boot metadata save failed still holds its control-socket + // listener, and releasing its device would tear the vGPU out from + // under a live VM. + hypervisorLive := m.vgpuHypervisorMayBeAlive(ctx, stored) if vgpuAssignmentMayBeLive(stored, m.nowUTC(), hypervisorLive) { if stored.GPUDevicePath != "" { protected[stored.GPUDevicePath] = struct{}{} @@ -97,6 +101,21 @@ func (m *manager) reconcileVGPUAssignments(ctx context.Context) (map[string]stru return protected, nil } +// vgpuHypervisorMayBeAlive fails closed and records why reconciliation could +// not establish ownership, so retained capacity remains diagnosable. +func (m *manager) vgpuHypervisorMayBeAlive(ctx context.Context, stored *StoredMetadata) bool { + live, err := hypervisorMayBeAlive(stored.HypervisorProcessIdentity, stored.SocketPath) + if err == nil { + return live + } + logger.FromContext(ctx).WarnContext(ctx, "preserving vGPU assignment because hypervisor liveness is uncertain", + "instance_id", stored.Id, + "device_path", storedVGPUDevicePath(stored), + "error", err) + m.recordVGPUReconcileLivenessUncertain(ctx) + return true +} + // releaseStaleVGPUAssignment retries a release that previously failed, under // the instance lock. Liveness is re-verified after locking so a concurrent // start or restore keeps its assignment. A failed release only logs and keeps @@ -118,7 +137,7 @@ func (m *manager) releaseStaleVGPUAssignment(ctx context.Context, id string) { if path == "" { return } - hypervisorLive := hypervisorMayBeAlive(stored.HypervisorProcessIdentity, stored.SocketPath) + hypervisorLive := m.vgpuHypervisorMayBeAlive(ctx, stored) if vgpuAssignmentMayBeLive(stored, m.nowUTC(), hypervisorLive) { return } diff --git a/lib/instances/vgpu_reconcile_test.go b/lib/instances/vgpu_reconcile_test.go index 7a9b66731..6a2edf90a 100644 --- a/lib/instances/vgpu_reconcile_test.go +++ b/lib/instances/vgpu_reconcile_test.go @@ -107,6 +107,41 @@ func TestReconcileVGPUsSkipsVendorSweepWhenListingFails(t *testing.T) { assert.Equal(t, []bool{false, true}, sweeps, "the next pass retries the vendor sweep") } +func TestReconcileVGPUsReleasesStaleAssignment(t *testing.T) { + t.Parallel() + + var destroyed []devices.VGPUAssignment + m := &manager{ + paths: paths.New(t.TempDir()), + destroyVGPU: func(_ context.Context, assignment devices.VGPUAssignment) error { + destroyed = append(destroyed, assignment) + return nil + }, + reconcileVGPUDevices: func(context.Context, map[string]struct{}, bool) error { return nil }, + } + const id = "stopped-retained" + require.NoError(t, m.ensureDirectories(id)) + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: id, + GPUProfile: "NVIDIA L40S-2Q", + GPUFramework: devices.VGPUFrameworkVendorVFIO, + GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", + }})) + + m.ReconcileVGPUs(t.Context()) + + require.Len(t, destroyed, 1) + assert.Equal(t, devices.VGPUAssignment{ + Framework: devices.VGPUFrameworkVendorVFIO, + DevicePath: "/sys/bus/pci/devices/0000:82:00.4", + InstanceID: id, + }, destroyed[0]) + stored, err := m.loadMetadata(id) + require.NoError(t, err) + assert.Empty(t, stored.GPUDevicePath) + assert.Equal(t, "NVIDIA L40S-2Q", stored.GPUProfile, "profile is kept for the next start") +} + func TestReconcileVGPUsKeepsAssignmentWhenReleaseFails(t *testing.T) { t.Parallel() diff --git a/lib/instances/vgpu_sentinel.go b/lib/instances/vgpu_sentinel.go new file mode 100644 index 000000000..8f8c448e9 --- /dev/null +++ b/lib/instances/vgpu_sentinel.go @@ -0,0 +1,437 @@ +package instances + +import ( + "bufio" + "context" + "errors" + "fmt" + "io" + "log/slog" + "os" + "path/filepath" + "regexp" + "time" + + "github.com/kernel/hypeman/lib/devices" + "github.com/kernel/hypeman/lib/logger" + "go.opentelemetry.io/otel/metric" +) + +const ( + vgpuSentinelScanInterval = 5 * time.Second + vgpuSentinelMaxLineBytes = 64 * 1024 +) + +// Require a standalone guest-agent line so echoed marker text cannot count against a VF. +var ( + vgpuSentinelFailedPattern = regexp.MustCompile(`^\d{4}/\d{2}/\d{2} \d{2}:\d{2}:\d{2} \[guest-agent\] (HYPEMAN-GPU-INIT-FAILED ts=\S+ nvrm="NVRM: [^"\r\n]*RmInitAdapter failed![^"\r\n]*")\r?\n?$`) + vgpuSentinelOKPattern = regexp.MustCompile(`^\d{4}/\d{2}/\d{2} \d{2}:\d{2}:\d{2} \[guest-agent\] (HYPEMAN-GPU-INIT-OK ts=\S+)\r?\n?$`) +) + +// VGPUSentinelEvent classifies a guest-agent vGPU marker line. +type VGPUSentinelEvent int + +const ( + VGPUSentinelEventNone VGPUSentinelEvent = iota + VGPUSentinelEventInitFailed + VGPUSentinelEventInitOK +) + +// MatchVGPUSentinelLine extracts a complete guest-agent vGPU marker. +func MatchVGPUSentinelLine(line []byte) (marker string, event VGPUSentinelEvent) { + if match := vgpuSentinelFailedPattern.FindSubmatch(line); match != nil { + return string(match[1]), VGPUSentinelEventInitFailed + } + if match := vgpuSentinelOKPattern.FindSubmatch(line); match != nil { + return string(match[1]), VGPUSentinelEventInitOK + } + return "", VGPUSentinelEventNone +} + +type vgpuSentinelTarget struct { + instanceID string + vfAddress string + appLogPath string + assignedAt string +} + +type vgpuSentinelStore interface { + listVGPUSentinelTargets(ctx context.Context) ([]vgpuSentinelTarget, error) + getVGPUSentinelTarget(ctx context.Context, instanceID string) (vgpuSentinelTarget, bool, error) +} + +var _ vgpuSentinelStore = (*manager)(nil) + +type vgpuSentinelTail struct { + vfAddress string + assignedAt string + offset int64 + skippingLongLine bool +} + +type pendingVGPUSuccess struct { + target vgpuSentinelTarget + confirmed bool +} + +// VGPUSentinelController quarantines VFs reported as wedged by the guest agent. +type VGPUSentinelController struct { + store vgpuSentinelStore + log *slog.Logger + interval time.Duration + reportFailure func(devices.VFInitFailureReport) (devices.VFReportResult, error) + reportSuccess func(devices.VFInitSuccessReport) (devices.VFSuccessResult, error) + initFailures metric.Int64Counter + quarantines metric.Int64Counter + tails map[string]*vgpuSentinelTail + pendingSuccesses map[devices.VFInitSuccessReport]*pendingVGPUSuccess + discoverFramework func() (devices.VGPUFramework, []devices.VirtualFunction, error) + probeErrLoggedAt time.Time +} + +func NewVGPUSentinelController(manager Manager, meter metric.Meter, log *slog.Logger) (*VGPUSentinelController, error) { + if manager == nil { + return nil, fmt.Errorf("instance manager is nil") + } + if log == nil { + return nil, fmt.Errorf("logger is nil") + } + store, ok := manager.(vgpuSentinelStore) + if !ok { + return nil, fmt.Errorf("instance manager %T does not implement vgpuSentinelStore", manager) + } + + initFailures, err := meter.Int64Counter( + "hypeman_instances_vgpu_sentinel_init_failures_total", + metric.WithDescription("Total guest-reported vGPU driver init failures recorded by the sentinel (one per instance assignment)"), + ) + if err != nil { + return nil, err + } + quarantines, err := meter.Int64Counter( + "hypeman_instances_vgpu_sentinel_quarantines_total", + metric.WithDescription("Total VFs quarantined by the vGPU sentinel"), + ) + if err != nil { + return nil, err + } + _, err = meter.Int64ObservableGauge( + "hypeman_instances_vgpu_quarantined_vfs", + metric.WithDescription("Number of vGPU virtual functions currently quarantined"), + metric.WithInt64Callback(func(_ context.Context, o metric.Int64Observer) error { + o.Observe(int64(devices.TotalQuarantinedVFs())) + return nil + }), + ) + if err != nil { + return nil, err + } + _, err = meter.Int64ObservableGauge( + "hypeman_instances_vgpu_vf_health_store_unavailable", + metric.WithDescription("1 when the persisted VF health state failed to load; quarantine mutations are refused and vGPU placement is disabled until it is repaired"), + metric.WithInt64Callback(func(_ context.Context, o metric.Int64Observer) error { + if devices.VFHealthStoreUnavailable() { + o.Observe(1) + } else { + o.Observe(0) + } + return nil + }), + ) + if err != nil { + return nil, err + } + + return &VGPUSentinelController{ + store: store, + log: log.With("controller", "vgpu_sentinel"), + interval: vgpuSentinelScanInterval, + reportFailure: devices.ReportVFInitFailure, + reportSuccess: devices.ReportVFInitSuccess, + discoverFramework: devices.DiscoverVGPU, + initFailures: initFailures, + quarantines: quarantines, + tails: make(map[string]*vgpuSentinelTail), + pendingSuccesses: make(map[devices.VFInitSuccessReport]*pendingVGPUSuccess), + }, nil +} + +func (c *VGPUSentinelController) Run(ctx context.Context) error { + ticker := time.NewTicker(c.interval) + defer ticker.Stop() + vendorVFIO := false + for { + if !vendorVFIO { + var err error + vendorVFIO, err = c.probeVendorVFIO() + if err != nil && time.Since(c.probeErrLoggedAt) >= time.Minute { + c.log.Warn("vGPU sentinel framework discovery failed; retrying", "error", err) + c.probeErrLoggedAt = time.Now() + } + if err == nil && !vendorVFIO { + return nil + } + if vendorVFIO { + c.log.Info("vGPU sentinel controller started") + } + } + select { + case <-ctx.Done(): + return nil + case <-ticker.C: + c.scanOnce(ctx) + } + } +} + +func (c *VGPUSentinelController) probeVendorVFIO() (bool, error) { + framework, _, err := c.discoverFramework() + if err != nil { + return false, err + } + if framework == devices.VGPUFrameworkNone { + c.log.Info("vGPU sentinel controller exiting: host has no vGPU framework") + return false, nil + } + if framework != devices.VGPUFrameworkVendorVFIO { + c.log.Info("vGPU sentinel controller exiting: host vGPU framework is not vendor VFIO", "framework", string(framework)) + return false, nil + } + return true, nil +} + +func (c *VGPUSentinelController) scanOnce(ctx context.Context) { + for report, pending := range c.pendingSuccesses { + if c.processSuccess(ctx, report, pending) { + delete(c.pendingSuccesses, report) + } + } + + targets, err := c.store.listVGPUSentinelTargets(ctx) + if err != nil { + c.log.WarnContext(ctx, "vGPU sentinel scan failed to list instances", "error", err) + return + } + alive := make(map[string]struct{}, len(targets)) + for _, target := range targets { + alive[target.instanceID] = struct{}{} + c.scanTarget(ctx, target) + } + for id := range c.tails { + if _, ok := alive[id]; !ok { + delete(c.tails, id) + } + } +} + +func (c *VGPUSentinelController) scanTarget(ctx context.Context, target vgpuSentinelTarget) { + tail := c.tails[target.instanceID] + if tail == nil || tail.vfAddress != target.vfAddress || tail.assignedAt != target.assignedAt { + tail = &vgpuSentinelTail{vfAddress: target.vfAddress, assignedAt: target.assignedAt} + c.tails[target.instanceID] = tail + } + + err := scanSentinelLog(target.appLogPath, tail, func(line string, event VGPUSentinelEvent) { + if event == VGPUSentinelEventInitFailed { + c.handleFailure(ctx, target, line) + return + } + report := devices.VFInitSuccessReport{ + VFAddress: target.vfAddress, + InstanceID: target.instanceID, + AssignedAt: target.assignedAt, + } + pending := &pendingVGPUSuccess{target: target} + if c.processSuccess(ctx, report, pending) { + delete(c.pendingSuccesses, report) + } else { + c.pendingSuccesses[report] = pending + } + }) + if err != nil { + c.log.WarnContext(ctx, "vGPU sentinel scan failed to read app log", + "instance_id", target.instanceID, "error", err) + } +} + +// confirmAssignment rejects a marker only when the instance now holds a +// different VF assignment. Released assignments remain attributable to the +// assignment captured in the scan target. +func (c *VGPUSentinelController) confirmAssignment(ctx context.Context, target vgpuSentinelTarget) (bool, error) { + current, ok, err := c.store.getVGPUSentinelTarget(ctx, target.instanceID) + if err != nil { + return false, err + } + return !ok || (current.vfAddress == target.vfAddress && current.assignedAt == target.assignedAt), nil +} + +func (c *VGPUSentinelController) handleFailure(ctx context.Context, target vgpuSentinelTarget, line string) { + unchanged, err := c.confirmAssignment(ctx, target) + if err != nil { + c.log.WarnContext(ctx, "vGPU sentinel could not confirm assignment before recording an init failure", + "vf", target.vfAddress, "instance_id", target.instanceID, "error", err) + return + } + if !unchanged { + c.log.InfoContext(ctx, "vGPU sentinel skipping init failure: assignment changed during scan", + "vf", target.vfAddress, "instance_id", target.instanceID) + return + } + result, err := c.reportFailure(devices.VFInitFailureReport{ + VFAddress: target.vfAddress, + InstanceID: target.instanceID, + AssignedAt: target.assignedAt, + }) + if err != nil { + c.log.ErrorContext(ctx, "failed to record vGPU VF init failure", + "vf", target.vfAddress, "instance_id", target.instanceID, "error", err) + return + } + switch result.Outcome { + case devices.VFReportQuarantined: + c.log.ErrorContext(ctx, "quarantined wedged vGPU VF", + "vf", target.vfAddress, + "instance_id", target.instanceID, + "sentinel_line", line, + "failures", result.Failures, + "threshold", result.Threshold, + ) + c.initFailures.Add(ctx, 1) + c.quarantines.Add(ctx, 1) + case devices.VFReportRecorded: + c.log.WarnContext(ctx, "recorded vGPU VF init failure below quarantine threshold", + "vf", target.vfAddress, + "instance_id", target.instanceID, + "sentinel_line", line, + "failures", result.Failures, + "threshold", result.Threshold, + ) + c.initFailures.Add(ctx, 1) + } +} + +func (c *VGPUSentinelController) processSuccess(ctx context.Context, report devices.VFInitSuccessReport, pending *pendingVGPUSuccess) bool { + if !pending.confirmed { + unchanged, err := c.confirmAssignment(ctx, pending.target) + if err != nil { + c.log.WarnContext(ctx, "vGPU sentinel could not confirm assignment before clearing init failures", + "vf", report.VFAddress, "instance_id", report.InstanceID, "error", err) + return false + } + if !unchanged { + return true + } + pending.confirmed = true + } + + result, err := c.reportSuccess(report) + if err != nil { + c.log.WarnContext(ctx, "vGPU sentinel failed to clear recorded init failures", + "vf", report.VFAddress, "instance_id", report.InstanceID, "error", err) + return false + } + if result.Rescinded { + c.log.InfoContext(ctx, "rescinded vGPU VF quarantine after successful driver init from the triggering assignment", + "vf", report.VFAddress, "instance_id", report.InstanceID, "cleared", result.Cleared) + } else if result.Cleared > 0 { + c.log.InfoContext(ctx, "cleared recorded vGPU VF init failures after successful driver init", + "vf", report.VFAddress, "instance_id", report.InstanceID, "cleared", result.Cleared) + } + return true +} + +func scanSentinelLog(path string, tail *vgpuSentinelTail, handle func(string, VGPUSentinelEvent)) error { + f, err := os.Open(path) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + defer f.Close() + + info, err := f.Stat() + if err != nil { + return err + } + if info.Size() < tail.offset { + tail.offset = 0 + tail.skippingLongLine = false + } + if _, err := f.Seek(tail.offset, io.SeekStart); err != nil { + return err + } + + reader := bufio.NewReaderSize(f, vgpuSentinelMaxLineBytes) + for { + line, err := reader.ReadSlice('\n') + switch { + case err == nil: + tail.offset += int64(len(line)) + if tail.skippingLongLine { + tail.skippingLongLine = false + continue + } + marker, event := MatchVGPUSentinelLine(line) + if event != VGPUSentinelEventNone { + handle(marker, event) + } + case errors.Is(err, bufio.ErrBufferFull): + tail.offset += int64(len(line)) + tail.skippingLongLine = true + case errors.Is(err, io.EOF): + if tail.skippingLongLine { + tail.offset += int64(len(line)) + } + return nil + default: + return err + } + } +} + +func (m *manager) listVGPUSentinelTargets(ctx context.Context) ([]vgpuSentinelTarget, error) { + files, statErr, err := m.walkMetadataFiles() + if err != nil { + return nil, err + } + if statErr != nil { + logger.FromContext(ctx).WarnContext(ctx, "vGPU sentinel cannot stat some instance metadata; their VFs are not scanned", "error", statErr) + } + targets := make([]vgpuSentinelTarget, 0, len(files)) + for _, file := range files { + id := filepath.Base(filepath.Dir(file)) + target, ok, err := m.getVGPUSentinelTarget(ctx, id) + if err != nil { + logger.FromContext(ctx).WarnContext(ctx, "vGPU sentinel skipping unreadable instance metadata", "instance_id", id, "error", err) + continue + } + if ok { + targets = append(targets, target) + } + } + return targets, nil +} + +func (m *manager) getVGPUSentinelTarget(_ context.Context, instanceID string) (vgpuSentinelTarget, bool, error) { + meta, err := m.loadMetadata(instanceID) + if err != nil { + if errors.Is(err, ErrNotFound) { + return vgpuSentinelTarget{}, false, nil + } + return vgpuSentinelTarget{}, false, err + } + if meta.GPUFramework != devices.VGPUFrameworkVendorVFIO || meta.GPUDevicePath == "" { + return vgpuSentinelTarget{}, false, nil + } + assignedAt := "" + if meta.GPUAssignedAt != nil { + assignedAt = meta.GPUAssignedAt.UTC().Format(time.RFC3339Nano) + } + return vgpuSentinelTarget{ + instanceID: instanceID, + vfAddress: filepath.Base(meta.GPUDevicePath), + appLogPath: m.paths.InstanceAppLog(instanceID), + assignedAt: assignedAt, + }, true, nil +} diff --git a/lib/instances/vgpu_sentinel_test.go b/lib/instances/vgpu_sentinel_test.go new file mode 100644 index 000000000..ba9a0300b --- /dev/null +++ b/lib/instances/vgpu_sentinel_test.go @@ -0,0 +1,565 @@ +package instances + +import ( + "bytes" + "context" + "errors" + "log/slog" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/kernel/hypeman/lib/devices" + "github.com/kernel/hypeman/lib/logger" + "github.com/kernel/hypeman/lib/paths" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel/metric/noop" +) + +const testSentinelMarker = "HYPEMAN-GPU-INIT-FAILED ts=2026-08-20T15:04:05.123456789Z nvrm=\"NVRM: GPU 0000:e3:00.4: RmInitAdapter failed! (0x22:0x65:884)\"" +const testSentinelLine = "2026/08/20 15:04:05 [guest-agent] " + testSentinelMarker + "\n" +const testSentinelOKMarker = "HYPEMAN-GPU-INIT-OK ts=2026-08-20T15:04:05.123456789Z" +const testSentinelOKLine = "2026/08/20 15:04:05 [guest-agent] " + testSentinelOKMarker + "\n" + +func TestVGPUSentinelPattern(t *testing.T) { + t.Parallel() + + marker, event := MatchVGPUSentinelLine([]byte(testSentinelLine)) + assert.Equal(t, VGPUSentinelEventInitFailed, event) + assert.Equal(t, testSentinelMarker, marker) + + marker, event = MatchVGPUSentinelLine([]byte(testSentinelOKLine)) + assert.Equal(t, VGPUSentinelEventInitOK, event) + assert.Equal(t, testSentinelOKMarker, marker) + + for _, line := range []string{ + "[ 27.031415] NVRM: GPU 0000:e3:00.4: RmInitAdapter failed! (0x22:0x65:884)", + testSentinelMarker + "\n", + "customer output: " + testSentinelOKMarker + "\n", + "2026-08-20T15:04:05Z [INFO] [hypeman-init:entrypoint] cmd=[echo " + testSentinelMarker + "]\n", + strings.TrimSuffix(testSentinelLine, "\n") + " trailing customer output\n", + "2026/08/20 15:04:05 [guest-agent] HYPEMAN-GPU-INIT-FAILED ts=2026-08-20T15:04:05Z nvrm=\"something else\"\n", + "2026/08/20 15:04:05 [guest-agent] HYPEMAN-AGENT-READY ts=2026-08-20T15:04:05Z", + } { + _, event := MatchVGPUSentinelLine([]byte(line)) + assert.Equal(t, VGPUSentinelEventNone, event, "must not match %q", line) + } +} + +type fakeSentinelStore struct { + targets []vgpuSentinelTarget + listCalls int +} + +func (s *fakeSentinelStore) listVGPUSentinelTargets(context.Context) ([]vgpuSentinelTarget, error) { + s.listCalls++ + return s.targets, nil +} + +func (s *fakeSentinelStore) getVGPUSentinelTarget(_ context.Context, instanceID string) (vgpuSentinelTarget, bool, error) { + for _, target := range s.targets { + if target.instanceID == instanceID { + return target, true, nil + } + } + return vgpuSentinelTarget{}, false, nil +} + +func TestNewVGPUSentinelControllerRejectsUnsupportedManager(t *testing.T) { + _, err := NewVGPUSentinelController(&stubManager{}, noop.NewMeterProvider().Meter("test"), slog.New(slog.DiscardHandler)) + require.ErrorContains(t, err, "does not implement vgpuSentinelStore") +} + +func newTestSentinelController(t *testing.T, store vgpuSentinelStore) (*VGPUSentinelController, *[]devices.VFInitFailureReport) { + t.Helper() + counter, err := noop.NewMeterProvider().Meter("test").Int64Counter("test") + require.NoError(t, err) + var reported []devices.VFInitFailureReport + c := &VGPUSentinelController{ + store: store, + log: slog.New(slog.DiscardHandler), + interval: time.Hour, + reportFailure: func(report devices.VFInitFailureReport) (devices.VFReportResult, error) { + for _, previous := range reported { + if previous == report { + return devices.VFReportResult{Outcome: devices.VFReportUnchanged, Failures: 1, Threshold: 1}, nil + } + } + reported = append(reported, report) + return devices.VFReportResult{Outcome: devices.VFReportQuarantined, Failures: 1, Threshold: 1}, nil + }, + reportSuccess: func(devices.VFInitSuccessReport) (devices.VFSuccessResult, error) { + return devices.VFSuccessResult{}, nil + }, + initFailures: counter, + quarantines: counter, + tails: make(map[string]*vgpuSentinelTail), + pendingSuccesses: make(map[devices.VFInitSuccessReport]*pendingVGPUSuccess), + } + return c, &reported +} + +func TestVGPUSentinelControllerReportsFailureOnce(t *testing.T) { + t.Parallel() + + logPath := filepath.Join(t.TempDir(), "app.log") + store := &fakeSentinelStore{targets: []vgpuSentinelTarget{{ + instanceID: "instance-1", + vfAddress: "0000:e3:00.4", + appLogPath: logPath, + }}} + c, reported := newTestSentinelController(t, store) + ctx := context.Background() + + c.scanOnce(ctx) + require.NoError(t, os.WriteFile(logPath, []byte("booting\nnvidia driver loaded\n"), 0644)) + c.scanOnce(ctx) + assert.Empty(t, *reported) + + f, err := os.OpenFile(logPath, os.O_APPEND|os.O_WRONLY, 0644) + require.NoError(t, err) + _, err = f.WriteString("2026/08/20 15:04:05 [guest-agent] HYPEMAN-GPU-INIT-FAILED ts=2026-08-20T15:04:05Z nvrm=\"NVRM: GPU 0000:e3:00.4: RmInitAdapter fail") + require.NoError(t, err) + c.scanOnce(ctx) + assert.Empty(t, *reported) + + _, err = f.WriteString("ed! (0x22:0x65:884)\"\n") + require.NoError(t, err) + require.NoError(t, f.Close()) + c.scanOnce(ctx) + require.Len(t, *reported, 1) + assert.Equal(t, "0000:e3:00.4", (*reported)[0].VFAddress) + assert.Equal(t, "instance-1", (*reported)[0].InstanceID) + + appendSentinelLine(t, logPath) + c.scanOnce(ctx) + assert.Len(t, *reported, 1) +} + +func TestVGPUSentinelControllerProcessesSuccessAfterFailure(t *testing.T) { + tests := []struct { + name string + failure devices.VFReportResult + success devices.VFSuccessResult + }{ + { + name: "recorded", + failure: devices.VFReportResult{Outcome: devices.VFReportRecorded, Failures: 1, Threshold: 2}, + success: devices.VFSuccessResult{Cleared: 1}, + }, + { + name: "quarantined", + failure: devices.VFReportResult{Outcome: devices.VFReportQuarantined, Failures: 2, Threshold: 2}, + success: devices.VFSuccessResult{Cleared: 2, Rescinded: true}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + logPath := filepath.Join(t.TempDir(), "app.log") + require.NoError(t, os.WriteFile(logPath, []byte(testSentinelLine+testSentinelOKLine), 0644)) + store := &fakeSentinelStore{targets: []vgpuSentinelTarget{{ + instanceID: "instance-1", + vfAddress: "0000:e3:00.4", + appLogPath: logPath, + assignedAt: "2026-08-20T15:00:00Z", + }}} + c, reported := newTestSentinelController(t, store) + c.reportFailure = func(report devices.VFInitFailureReport) (devices.VFReportResult, error) { + *reported = append(*reported, report) + return tt.failure, nil + } + var successes []devices.VFInitSuccessReport + c.reportSuccess = func(report devices.VFInitSuccessReport) (devices.VFSuccessResult, error) { + successes = append(successes, report) + return tt.success, nil + } + + c.scanOnce(context.Background()) + require.Len(t, *reported, 1) + require.Len(t, successes, 1) + assert.Equal(t, "instance-1", successes[0].InstanceID) + }) + } +} + +func TestVGPUSentinelControllerRetriesFailedTallyClearAfterReassignment(t *testing.T) { + t.Parallel() + + logPath := filepath.Join(t.TempDir(), "app.log") + require.NoError(t, os.WriteFile(logPath, []byte(testSentinelOKLine), 0644)) + store := &fakeSentinelStore{targets: []vgpuSentinelTarget{{ + instanceID: "instance-1", + vfAddress: "0000:e3:00.4", + appLogPath: logPath, + assignedAt: "2026-08-20T15:00:00Z", + }}} + c, _ := newTestSentinelController(t, store) + var reports []devices.VFInitSuccessReport + c.reportSuccess = func(report devices.VFInitSuccessReport) (devices.VFSuccessResult, error) { + reports = append(reports, report) + if len(reports) == 1 { + return devices.VFSuccessResult{}, errors.New("persist failed") + } + return devices.VFSuccessResult{Cleared: 1}, nil + } + + c.scanOnce(context.Background()) + require.Len(t, c.pendingSuccesses, 1) + + store.targets[0].vfAddress = "0000:e3:00.5" + store.targets[0].assignedAt = "2026-08-20T16:00:00Z" + require.NoError(t, os.WriteFile(logPath, []byte("new boot\n"), 0644)) + c.scanOnce(context.Background()) + + require.Len(t, reports, 2) + assert.Equal(t, "0000:e3:00.4", reports[1].VFAddress) + assert.Empty(t, c.pendingSuccesses) +} + +func TestVGPUSentinelControllerSkipsInitOKOnChangedAssignment(t *testing.T) { + t.Parallel() + + logPath := filepath.Join(t.TempDir(), "app.log") + require.NoError(t, os.WriteFile(logPath, []byte(testSentinelOKLine), 0644)) + store := &fakeSentinelStore{targets: []vgpuSentinelTarget{{ + instanceID: "instance-1", + vfAddress: "0000:e3:00.5", + appLogPath: logPath, + assignedAt: "2026-08-21T00:00:10Z", + }}} + c, _ := newTestSentinelController(t, store) + var cleared []devices.VFInitSuccessReport + c.reportSuccess = func(report devices.VFInitSuccessReport) (devices.VFSuccessResult, error) { + cleared = append(cleared, report) + return devices.VFSuccessResult{Cleared: 1}, nil + } + + c.scanTarget(context.Background(), vgpuSentinelTarget{ + instanceID: "instance-1", + vfAddress: "0000:e3:00.4", + appLogPath: logPath, + assignedAt: "2026-08-21T00:00:00Z", + }) + assert.Empty(t, cleared) + assert.Empty(t, c.pendingSuccesses) +} + +func TestVGPUSentinelControllerAppliesInitOKFromReleasedAssignment(t *testing.T) { + t.Parallel() + + logPath := filepath.Join(t.TempDir(), "app.log") + require.NoError(t, os.WriteFile(logPath, []byte(testSentinelOKLine), 0644)) + c, _ := newTestSentinelController(t, &fakeSentinelStore{}) + var cleared []devices.VFInitSuccessReport + c.reportSuccess = func(report devices.VFInitSuccessReport) (devices.VFSuccessResult, error) { + cleared = append(cleared, report) + return devices.VFSuccessResult{Cleared: 1}, nil + } + + c.scanTarget(context.Background(), vgpuSentinelTarget{ + instanceID: "instance-1", + vfAddress: "0000:e3:00.4", + appLogPath: logPath, + assignedAt: "2026-08-21T00:00:00Z", + }) + require.Len(t, cleared, 1) + assert.Equal(t, "2026-08-21T00:00:00Z", cleared[0].AssignedAt) +} + +func TestVGPUSentinelControllerRetriesFailedQuarantine(t *testing.T) { + t.Parallel() + + logPath := filepath.Join(t.TempDir(), "app.log") + require.NoError(t, os.WriteFile(logPath, []byte(testSentinelLine), 0644)) + store := &fakeSentinelStore{targets: []vgpuSentinelTarget{{ + instanceID: "instance-1", + vfAddress: "0000:e3:00.4", + appLogPath: logPath, + }}} + c, reported := newTestSentinelController(t, store) + realReport := c.reportFailure + c.reportFailure = func(devices.VFInitFailureReport) (devices.VFReportResult, error) { + return devices.VFReportResult{}, errors.New("persist failed") + } + + c.scanOnce(context.Background()) + assert.Empty(t, *reported) + + c.reportFailure = realReport + appendSentinelLine(t, logPath) + c.scanOnce(context.Background()) + assert.Len(t, *reported, 1) +} + +func TestVGPUSentinelControllerRescansNewAssignment(t *testing.T) { + t.Parallel() + + logPath := filepath.Join(t.TempDir(), "app.log") + require.NoError(t, os.WriteFile(logPath, []byte(testSentinelLine), 0644)) + store := &fakeSentinelStore{targets: []vgpuSentinelTarget{{ + instanceID: "instance-1", + vfAddress: "0000:e3:00.4", + appLogPath: logPath, + assignedAt: "2026-08-20T15:00:00Z", + }}} + c, reported := newTestSentinelController(t, store) + + c.scanOnce(context.Background()) + require.Len(t, *reported, 1) + + require.NoError(t, os.WriteFile(logPath, []byte(testSentinelLine), 0644)) + store.targets[0].vfAddress = "0000:e3:00.5" + store.targets[0].assignedAt = "2026-08-20T16:00:00Z" + c.scanOnce(context.Background()) + require.Len(t, *reported, 2) + assert.Equal(t, "0000:e3:00.5", (*reported)[1].VFAddress) +} + +func TestVGPUSentinelControllerDropsStaleTails(t *testing.T) { + t.Parallel() + + logPath := filepath.Join(t.TempDir(), "app.log") + require.NoError(t, os.WriteFile(logPath, []byte("booting\n"), 0644)) + store := &fakeSentinelStore{targets: []vgpuSentinelTarget{{ + instanceID: "instance-1", + vfAddress: "0000:e3:00.4", + appLogPath: logPath, + }}} + c, _ := newTestSentinelController(t, store) + + c.scanOnce(context.Background()) + require.Contains(t, c.tails, "instance-1") + + store.targets = nil + c.scanOnce(context.Background()) + assert.NotContains(t, c.tails, "instance-1") +} + +func TestVGPUSentinelControllerSkipsFailureOnChangedAssignment(t *testing.T) { + t.Parallel() + + logPath := filepath.Join(t.TempDir(), "app.log") + require.NoError(t, os.WriteFile(logPath, []byte(testSentinelLine), 0644)) + store := &fakeSentinelStore{targets: []vgpuSentinelTarget{{ + instanceID: "instance-1", + vfAddress: "0000:e3:00.5", + appLogPath: logPath, + assignedAt: "2026-08-21T00:00:10Z", + }}} + c, reported := newTestSentinelController(t, store) + + stale := vgpuSentinelTarget{ + instanceID: "instance-1", + vfAddress: "0000:e3:00.4", + appLogPath: logPath, + assignedAt: "2026-08-21T00:00:00Z", + } + c.scanTarget(context.Background(), stale) + assert.Empty(t, *reported) + + store.targets = nil + c.tails["instance-1"] = &vgpuSentinelTail{vfAddress: stale.vfAddress, assignedAt: stale.assignedAt} + c.scanTarget(context.Background(), stale) + require.Len(t, *reported, 1) + assert.Equal(t, "0000:e3:00.4", (*reported)[0].VFAddress) +} + +func TestListVGPUSentinelTargetsSkipsUnstattableMetadata(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("root bypasses directory permissions") + } + + m := &manager{paths: paths.New(t.TempDir())} + assigned := time.Now().UTC() + for _, id := range []string{"readable", "unreadable-a", "unreadable-b"} { + require.NoError(t, m.ensureDirectories(id)) + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: id, + GPUFramework: devices.VGPUFrameworkVendorVFIO, + GPUDevicePath: "/sys/bus/pci/devices/0000:e3:00.4", + GPUAssignedAt: &assigned, + }})) + } + for _, id := range []string{"unreadable-a", "unreadable-b"} { + instanceDir := filepath.Dir(m.paths.InstanceMetadata(id)) + require.NoError(t, os.Chmod(instanceDir, 0o000)) + t.Cleanup(func() { _ = os.Chmod(instanceDir, 0o755) }) + } + + files, err := m.listMetadataFilesStrict() + require.Error(t, err) + assert.ErrorContains(t, err, "unreadable-a") + assert.ErrorContains(t, err, "unreadable-b") + require.Len(t, files, 1) + + var logs bytes.Buffer + ctx := logger.AddToContext(context.Background(), slog.New(slog.NewTextHandler(&logs, nil))) + targets, err := m.listVGPUSentinelTargets(ctx) + require.NoError(t, err) + require.Len(t, targets, 1) + assert.Equal(t, "readable", targets[0].instanceID) + assert.Contains(t, logs.String(), "vGPU sentinel cannot stat some instance metadata; their VFs are not scanned") + assert.Contains(t, logs.String(), "unreadable-a") + assert.Contains(t, logs.String(), "unreadable-b") +} + +func TestVGPUSentinelControllerRunExitsWhenHostIsNotVendorVFIO(t *testing.T) { + tests := []struct { + name string + framework devices.VGPUFramework + firstError bool + wantProbes int + wantScans int + wantLogText string + }{ + { + name: "missing framework", + wantProbes: 1, + wantLogText: "host has no vGPU framework", + }, + { + name: "discovery retry resolves to mdev", + framework: devices.VGPUFrameworkMdev, + firstError: true, + wantProbes: 2, + wantScans: 1, + wantLogText: "host vGPU framework is not vendor VFIO", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + store := &fakeSentinelStore{} + c, _ := newTestSentinelController(t, store) + c.interval = time.Millisecond + var logs bytes.Buffer + c.log = slog.New(slog.NewTextHandler(&logs, nil)) + var probes int + c.discoverFramework = func() (devices.VGPUFramework, []devices.VirtualFunction, error) { + probes++ + if tt.firstError && probes == 1 { + return devices.VGPUFrameworkNone, nil, errors.New("transient sysfs error") + } + return tt.framework, nil, nil + } + + done := make(chan error, 1) + go func() { done <- c.Run(context.Background()) }() + select { + case err := <-done: + require.NoError(t, err) + case <-time.After(10 * time.Second): + t.Fatal("Run did not exit") + } + assert.Equal(t, tt.wantProbes, probes) + assert.Equal(t, tt.wantScans, store.listCalls) + assert.NotContains(t, logs.String(), "vGPU sentinel controller started") + assert.Contains(t, logs.String(), tt.wantLogText) + }) + } +} + +func scanForSentinel(path string, tail *vgpuSentinelTail) (string, VGPUSentinelEvent, error) { + var marker string + var event VGPUSentinelEvent + err := scanSentinelLog(path, tail, func(found string, foundEvent VGPUSentinelEvent) { + marker, event = found, foundEvent + }) + return marker, event, err +} + +func TestScanForSentinelMatchesOnIntactRepeatAfterSplitMarker(t *testing.T) { + t.Parallel() + + logPath := filepath.Join(t.TempDir(), "app.log") + split := "2026/08/20 15:04:05 [guest-agent] HYPEMAN-GPU-INIT-FAILED ts=2026-08-20T15:04:05Z nvrm=\"NVRM: GPU 0000:e3:0\n" + + "[ 27.031415] NVRM: GPU 0000:00:03.0: RmInitAdapter failed! (0x22:0x65:884)\n" + + "0.4: RmInitAdapter failed! (0x22:0x65:884)\"\n" + require.NoError(t, os.WriteFile(logPath, []byte(split), 0644)) + + tail := &vgpuSentinelTail{} + _, event, err := scanForSentinel(logPath, tail) + require.NoError(t, err) + assert.Equal(t, VGPUSentinelEventNone, event, "neither a split marker nor the raw kernel line may match") + + appendSentinelLine(t, logPath) + line, event, err := scanForSentinel(logPath, tail) + require.NoError(t, err) + require.Equal(t, VGPUSentinelEventInitFailed, event, "the intact repeat must match") + assert.Contains(t, line, "HYPEMAN-GPU-INIT-FAILED") +} + +func TestScanForSentinelDiscardsOversizedLines(t *testing.T) { + t.Parallel() + + logPath := filepath.Join(t.TempDir(), "app.log") + tail := &vgpuSentinelTail{} + + huge := strings.Repeat("x", vgpuSentinelMaxLineBytes) + testSentinelLine + require.NoError(t, os.WriteFile(logPath, []byte(huge), 0644)) + line, event, err := scanForSentinel(logPath, tail) + require.NoError(t, err) + assert.Equal(t, VGPUSentinelEventNone, event, "a marker inside an oversized line must not match") + assert.Empty(t, line) + + appendSentinelLine(t, logPath) + line, event, err = scanForSentinel(logPath, tail) + require.NoError(t, err) + require.Equal(t, VGPUSentinelEventInitFailed, event) + assert.Contains(t, line, "RmInitAdapter failed!") +} + +func TestScanForSentinelDiscardsOversizedLineTailAcrossScans(t *testing.T) { + t.Parallel() + + logPath := filepath.Join(t.TempDir(), "app.log") + tail := &vgpuSentinelTail{} + + require.NoError(t, os.WriteFile(logPath, []byte(strings.Repeat("x", vgpuSentinelMaxLineBytes+10)), 0644)) + _, event, err := scanForSentinel(logPath, tail) + require.NoError(t, err) + assert.Equal(t, VGPUSentinelEventNone, event) + assert.True(t, tail.skippingLongLine) + + appendSentinelLine(t, logPath) + _, event, err = scanForSentinel(logPath, tail) + require.NoError(t, err) + assert.Equal(t, VGPUSentinelEventNone, event, "the tail of an oversized line must not match") + assert.False(t, tail.skippingLongLine) + + appendSentinelLine(t, logPath) + line, event, err := scanForSentinel(logPath, tail) + require.NoError(t, err) + require.Equal(t, VGPUSentinelEventInitFailed, event) + assert.Contains(t, line, "RmInitAdapter failed!") +} + +func TestScanForSentinelResetsSkipStateOnTruncatedLog(t *testing.T) { + t.Parallel() + + logPath := filepath.Join(t.TempDir(), "app.log") + tail := &vgpuSentinelTail{} + + require.NoError(t, os.WriteFile(logPath, []byte(strings.Repeat("x", vgpuSentinelMaxLineBytes+10)), 0644)) + _, _, err := scanForSentinel(logPath, tail) + require.NoError(t, err) + require.True(t, tail.skippingLongLine) + + require.NoError(t, os.WriteFile(logPath, []byte(testSentinelLine), 0644)) + line, event, err := scanForSentinel(logPath, tail) + require.NoError(t, err) + require.Equal(t, VGPUSentinelEventInitFailed, event) + assert.Contains(t, line, "RmInitAdapter failed!") + assert.False(t, tail.skippingLongLine) +} + +func appendSentinelLine(t *testing.T, path string) { + t.Helper() + f, err := os.OpenFile(path, os.O_APPEND|os.O_WRONLY, 0644) + require.NoError(t, err) + _, err = f.WriteString(testSentinelLine) + require.NoError(t, err) + require.NoError(t, f.Close()) +} diff --git a/lib/oapi/oapi.go b/lib/oapi/oapi.go index ba5df2db0..f2948267f 100644 --- a/lib/oapi/oapi.go +++ b/lib/oapi/oapi.go @@ -1018,6 +1018,9 @@ type GPUProfile struct { // GPUResourceStatus GPU resource status. Null if no GPUs available. type GPUResourceStatus struct { + // AllocatableSlots Free slots eligible for placement, matching admission control (excludes quarantined VFs; 0 while VF health state is unavailable) + AllocatableSlots int `json:"allocatable_slots"` + // Devices Physical GPUs (only in passthrough mode) Devices *[]PassthroughDevice `json:"devices,omitempty"` @@ -1027,10 +1030,13 @@ type GPUResourceStatus struct { // Profiles Available vGPU profiles (only in vGPU mode) Profiles *[]GPUProfile `json:"profiles,omitempty"` + // QuarantinedSlots VFs quarantined after guest driver init failures (vGPU mode only). May overlap used_slots until the affected instance releases its VF. + QuarantinedSlots int `json:"quarantined_slots"` + // TotalSlots Total slots (VFs for vGPU, physical GPUs for passthrough) TotalSlots int `json:"total_slots"` - // UsedSlots Slots currently in use + // UsedSlots Slots currently in use. Includes quarantined VFs that are still assigned, so this can overlap quarantined_slots. UsedSlots int `json:"used_slots"` } @@ -18991,249 +18997,252 @@ var swaggerSpec = []string{ "b/uvJmpekx9IbJSZ6KGUx7UaT/b9dhfRy7N3Z01jKqouIX90S3NaAe+zHO0yyZmp++TZVLCtx2gR1wwe", "j+msj34kUvXIZMKFOrCBLOCQ8mq+4gUSJM41PfjQZJwhSccJnNGiVy3w61/0hAAIZJxPJkRUUQeeh2Rp", "7+1RGjDu/aSfI/NCAXtw+mNVntZye1ut/axCDaC2T3BE2XSz9XYHDIK1aazDFXx59u6NLRvUhLisl7Io", - "LWTAlvvo16Kwll5qWaIq9QOWwjpyX0Mi2dlsIWmEE9OiqYpBmW/gg9PQWkI/Kz+0ptCAnB6ux+ZOHtqY", - "T7Mczv35m97J6/dbaUzm3cqYIJBvxhOix73psae5QzMos94qXGneZGkxhCHbnlhvrQqW0XqRPAYRWB3F", - "FU5GMuGhmKG3+iGCh2jj/U8mHVmPoIuyylbq332oIZ++nwRPDCDYN3R7Dh3WTbaVAx7UXesQaJ3q9Cqd", - "ho6KSaValrGWiyHyy+pG88v1BfhMI839Hrlsr5pR3QqyyCSFGbxGFwaBzKfGam5NM5JkWGBFkkUtHrMK", - "qE6WPdrkmkQ3SDd7oV//ZIpo5IKM1EwQOeNJNQ5it7tciFVCzPGc2NpTZk6e4V9xlGJxCTexE+RRzswK", - "VEPWd9fhy8yUym4wqZ/fvj0z2r0iYo6TetKDXPLwH5MEL9CYqCtCmJsKlgj7Ma/1pFHZUP9HqFFGBOXV", - "NezsBvo9N3HQaCpwRJD5ypVxtlsiIb6o7VLaXgKIhVFEpGzY3+1V+2s/neRJuz0ODWt7bdXz6CYb/Pbo", - "zNWxKeoEu2XeWV7lMyJ65si5gsGrt3ZHrq6o5LpiBoh2SWAYEyixZDNN/ERQF98EJaT055XkS485SD9d", - "0PYDp8AsVdec8w+tpMv6cQ858lPM4lC9ZZNgYFLzp4DXBVHOIgfRj8Ym+MmgBRg9wM+TEATHlBEpa3nN", - "US6STrfTm9hZHWxtJTzCCUAW7u1uP9taHUa6Mn7YhkuNYrpKv3RBVSbsxmXRGrw3mHSVJLZwlrWwwJl1", - "XHM/AHtajleEGsv6bvMkPBfAPhgspWhf40i5qnBgkqu4XLF/bAFYuDIfaDA1hV60qP3cP5+DoA88w2pW", - "Jf+tJdqHuBiIadQ0YswdtXU0RP4xKFFxESr9xYWyGX9j4oIei/vQhRQ6CN2Kc27wzJ/lk/393f11fAiY", - "Te2Y23MXmKp5u5rTTLzltufXNgDhWVWZwx3p1dVL9bqsoSnNEZdIavWC8oywG63n/t7uzs3Ws+1ETlxY", - "WI0vhSBhjk6PjUwUcaYwZUSglCgcY4WrTAZsWZrLQG0ZTFJIC5p8v5q1NMRP+Bgvty2M9aW87w018t44", - "SOgUMzrRDNm+6fcsZ3hn/8mBqeQZk8ne/pN+v39T5IsXJdRFq63YMkF6HghGX84+bx/uAOCizVz+6Jwd", - "vv1ZM7JcCnNpbckxZQfev4t/lg/gD/PPMWVhYIw2xV/pZKnoazUeLbfIwyQ+QGV9byf3tIkPajBGQ3Qy", - "oPEEYeYqUZp3hydX0Ditlg24QW7wilxZLa68ZsmicW1uXce1rHCuvPqtfrZZi1qu9ONq/7ozd8E7tk8D", - "QF2UuV32rN+qULFcWctxqdRXRlhRvTFJzF8RZwDsGyrlWLki3bMWlcDgGrElv4ou/R+L3r0fj/yBeL+7", - "SmLeT7am44cbhsSsFEj/tiyHrudCThxdc5jDtsfiVmhbP9fi0AVjwR/4LrxN2Fi199fT//r9/8izp3/f", - "/v3V+/f/PX/5X8e/0v9+n5y9/iyMk9UQhA+KI/jFoANNTXgfP7AtKZ1iFQVsdFr9a1hh+8RYHFQ0g4qf", - "aEwOhqyHXlFFhKkfV0t+HHbQBgFNCb7S4i4UxzF5Z5v64zPj0dQf/+HE4E/1NmKblC7shhRYIzIfxzzF", - "lG0O2ZDZtpCbiAS9QP8VowhnpggcZUjrvws0FlCxz7qYys676A+cZZ82h8xW6Tdo2hmGomeTIuuLOQex", - "HZUJg7WvkwJ2wmQkDllxWxcYfMbP2C8B+ylJ6jlDDYuyWn+zmtOzQQitEPJZ9EZCURpQQQrK1mRUJNqg", - "Z4PNZX1ujY5R0NAK8rOOeJPweJiHzMVNSZLHJKYR8BWXJzizmaRFiqahNGvEywS/XsDevDHJazHCuZpp", - "XhTZxPqI80tKurClXXCHQeQHfGn8+TOe9caL3oxnBcgCFibaBRuPeFXJ/j89O9HeeyLoxPYUzJXXJBIQ", - "OuHI2JmZlMPCurA0sbemngfTos+c2NdNYRhpaoGYUG6VC+bKVRAoWgloHgX1kZBM/j2KEgpWJznjeRKj", - "GQDuKd1MCDOvMyhSZvA4ismk/u9qSMPO/hPQYN2/d3daZ6yapVtFZXkS0GlTx/pacGzDJmEARjwYOUP4", - "miAkfQNaPy7YKRSH/54j11B54gpGYrxTJodO2uoHifSy5zaDaU72GFjIkBG2p6nNfbR0CitZUy1aMNEJ", - "8FnSAqzkhcnWfPvqHCkiUpc/vxHp3YFTYpApelTK3BbiOjw6fbHZ7wSBliquKtiqlVlV1UEHsBZstEJT", - "EEZpo8Ep6aKTY8iWtddKqYtBesNPXKDE3IrlZXQAYB1Vcw82JfxOjq0AmizKkAcjtgw7m67FrH69HaA3", - "hQqIi6EUeY8lbbkmy8sEmrUBcCb3Yqn1Wpos+Mes+mfvY8i0gPqGhhcDWGTj/dXe5uigp/RFVbOQ3fhC", - "8qNQGu1f3t5/aVjlLy+j795MRrc+4FE2wzJE3TPfqwkvLe2770ausnvRHDNU6XckafBs/c1VefGuIUX0", - "PVf5PIQjut/b3n67vXdz891NEXGrUFgeTF4BitsezfYuUGEDGK9UjRqDy5F+bEPJnV3k/SmaYcm+U/Cw", - "Zh3Z3n3axigBvbYNy/YDsvnEDKngUg5XqwgnNghjlzRJjAAj6ZThBD1HG+cnL385efVqE/XQ69en9a1Y", - "9UVwf24Bjgu3AKyjyTALQCtVIAFQkTv49u0rOFwJgfQLI4df3h4yd61psQWErhvcy7N34PjHcuQCN5tz", - "FXGZ70uuqVRyGVWtVfzz50D2mk/bFfl3kzRtlLX+V+P+/lwBpg3C5G3eAWCvC15fWs4HwLJ9yCTBrw9H", - "dyXy7efC11o7wx2h1zZeaSHk1xoiwn7T7XZ7HNo7GU4FUCbEtnwJx2Vw3xr4tduhgezVQ6kvHhKjk7Oy", - "0lPpjHDN1+b0fKe//eQZFCvdHrRh7CmOVvR9enjUvvPBjrllDvD4IIoPQGG/rc/KErZRQXByhRdQ7M8s", - "7bBjLkxPu/WOrVUkW8XXLOPr3g5Oty7GNQDmgjjrApfkKF1Za6RFemIdNC3NLSRkSpOEShJxFsuqjDzD", - "EsnMIKGamhuFBD9kMMAuKkofg5SCcBSJvDQ9Wunayvt5Zuke6n5mnGkdAID/fyELiVIKTtCiewh9lKjI", - "gomHbEO4jKkiNQpKfsb6B8g/6NrI9lgPjSqoLaI/GDI5y5VmYpt9dMSZzFMirFUWjSl4jDaRzI1KC+OF", - "1VhohilpTMSQ6dcCWKt/FOrJwZPBYDDodgpNblf/exCipjt1fvYtlrDJ+QW0P2ZRhQFGUOQM5Swmoqj/", - "TQw51EPkbug4/UwQYfd5O/HKfl7KVeGDuQ5zuB2Y8OciuMJQG/RziA69hXK+f3sRvVXOkZNfbbaR/Wp0", - "kwgGgiKeJ7HW+Mb6tjMGORJbM6QkynBn8y6V6J2pvVmdug09Vhz9nhOxQO9PTythD4JMNA9oN3HgEg37", - "wLMbbcPOGhvJ2tHcxL3s4d3eB8ZtXVLxJMQvjmjrexhdErSh0Iphq6I4r7Kvaa0ymEVCmdknTTQrJhhK", - "kRmVgZF+q3IhJxbf0YrS2MlDBm+6gH8onfpyIbfGudzKIrpl82+2AJvjGWBz7AWTp2MyH+V5SDXSjxwY", - "y7t3J8doA34BbFlIoawSMMZPtp8Nnj3vPRtvP+ntxYPtHt7efdLb2ceDyW70dHd7Z3dFIkyLbLrbJ8gF", - "NeZAHHMRtT5y0fOhoOam3IWabGLjsa8oi/lV5foLBsj6vdvg23XdL4fWtx5CMCEnwVIZ80UDJzuFS55E", - "um0TkG4zNotSS2FD55O3g+3Ptf7A4BruiLciZ8atajAGChdC6g3Y36zqOG/H8mFALvFl3Wr5nbdftMHB", - "/vOD/c9dNJe8sW6MdXK6x81tighzGMy17BCXoejZkZyBsmNlImPVt8kknW6nyHeBv0EYqMVSF49bJXE1", - "HdhumI2sulYakqdPKvoKRKoYDL74QEsqTh+Big5Fir4WgY4SnsfIs8UZSDLww514uotuBtxi1kRnIEZN", - "MobWcQDTGipHUKYZMfgfdSM20/oAvYR34RFOjVpnB2Hql/iuNxwvTLyMPl+ua6NkrR7yudWv4ButbCH9", - "L5i2XgZrsl3dhJHODtCvHL4ptD3G67Zf8zqoWcuv1+3EGxau2yFnQGdW1DxAPxXiZSGgWoF0QxL758gy", - "rBKwZrMCG2B3vKOppdw5LwW+2zEr2ul23EJBqvxy0vy7kuqXzp9PiqFAMoITOMtljnCuaGJhumEmVCoa", - "SZs8oje3SeyxpZVIPDLKU1NMqkk8tQpW8ZGTqt6fog1AYvwLsoZt/a/NIn61ctftPN97/uTpzvMnrfCW", - "ygGuF42PIC16eXBr5eQoy0fWNtI09aOzd8b2ERmrQhH78v7Uh7fIBNesR8/cNeh3/rz/3IeZink+TjzH", - "osWkM6i2sGFBJLWCFzXEQf5OkzmdTNjvH6PLnb8Lmm5fP5E74+0G+FzTUdjsduIHFyzZqMm4Z8okhZGA", - "gKCEbATLekMkzACdE4WAfnoIR6DeFNnMluQcpJZd8SBh7e3u7j57ur/Tiq7s6LyDMwIjXOBStiPwjhi8", - "iTbenJ+jLY/gTJsOUwIQzplVfcPnDNkax4OqQNrfHuyGqKTh4i6pxrY9TxuX/L1VH+2k7KJDUnahWi6d", - "8uBq7+4Onu7tP9tvd4yteXgkrldzGJeyZJbHAvH7O78B0uTbwzMECcETHFVtOy5C7EajUjcaFRSRMODv", - "NxjYs6dP9vd2d7bbob6Fgk4snmHlwFZ5V+DQBYgisBuBpVhmvd2m2yIkThkCe0OiBNP0MHIpFrXbx4C8", - "j4R5rdyENheD1cCXLq4W37YybhUmK5OgY0QDLlDOitIi/fUu2S/iWW3m2uZ6WM/VQ2k5TK+ehScyJdRu", - "sZSZIHPKc/kFGuLK5MxOEs7Fjb5tUljeEJknythsqETvT78DnqJpDUlFsqoOZalxBYjTLSd3o/NcIZEw", - "kTctVqvdaLP1qybcbTi13VWAGhVu0AidFmvOlbP1wZ9HOIlyKKaDi/3UswIMMIAEyLJkYWL7k4RzhqIZ", - "ZuAkER7iEZrxJO4HI2H1k9EkGFXBr1DCDejzJSGZrTNjBqE/0yIMnRO04VdYM6RUq3u6nxomYyuJVKlx", - "Pw0XcMQylKxWpMLr9cSKe3jE5pOKJTThUwlKoYKshX4dBj/DwiQjYGbqJs1To0sGAq4DQ6wx89CNam5S", - "PrEKrhU5INHcrCSOBJcSkYROoUbP+9Na/vKKnLcii3l9QGd1sC1I1zg0A1eZQcNqXV4tdD8G8nk+54YE", - "GoacwRWhks44mWKWQ+UZj5CtIb7fOhxyxqUaFbhUNxysVCMoJ5ELUqLlFVn3hT3IvRO8Fx1ru81y2bjj", - "W329RFXhppoG2MxTgysaXq1uQYMhMl5G5loJBlaii9WhpG4CVlfWH6ASWqUebBnagJwXjy15EHSbbYJk", - "wiqr7mdJW7XVQV/tDc7bwrqtRnE7w2p2wiY8gP1xA8+ps0TbaNWMiJRCQRUUE0ZJ7HTJwoVqTV2QMJ5I", - "guKc2JUz8qnAdsGxOd7gs2LORkbZtMbr6x22MQ+bMayuNgH92hfbhDvJcELtW5HDWpl4RYlwmVrbKgiU", - "ylHYnbXcsCDTPMECWUDGNkOWizSh7LJN63KRjnlCI6Q/qPvFJzxJ+NVIP5I/wFw2W81OfzBqKk10bgZn", - "8wLNhtT6Lafwg57lZi0rGSwxW+b7LXCMtokeC0aK/0QTYtH93jF67RF6FY59b2fQlC3f0GglT34ZGfKm", - "nNuSbPDE5zKQW7hSynFFlUhsMfKN2JPl0tR3aXErORBW5wK8nUenmjjyedAkR4Zf14BJ0JhA3o+b2jLX", - "aMEW20wlWFoilzP0dz6uGkTbhv0GCpZtsBIiQ5BJML4fdnSlQdq8sbQm3u7eBIMC2KqeKHx0Q2iHdaXd", - "yviqJn7yZqnK2YzYJaNujqbiWYsKHi7+o4AvsL22xzGo16MLxCsDSo1UC6jsCuV0Fl6RRYnGXAhAoNYS", - "DmduNgC7omUevdYO9wq9nZEFEiTFlA0ZZYWRFMDUCGJkToSXJcuFVrKmJO6jv3kqHmB2p5laWDB4MJ5/", - "JxG/YsUYh8wfpG48l7qdQ2YsiyLPVKVcpG4WtD5NKJC1DE4wJaB+IlUzNBFEzvy5h2pmahnviou4sRjR", - "ArlXoMYN+FiR4peE+aysaCaoGpqGRuar5Sg+U/AWnlr9E1Vq0KJ6jdnV/eWSiLCQWEypeKVV6Ip3VDzl", - "xIDAACIK1Ba0fxkWX6CgtMA8KZv/q2uy/OmsaLz6W+01D9fEwQwfGrNt0AQbmTSeWrBP1ZO2NlQF0uBW", - "odks+xLQhguhdgVaqpKAVyil1T3ZLhOvnizgRrMlSVTtfe/Z/tMnLSvVfJazzqB3fWnX3Dxd4ZJr2KnT", - "Nn6fZ/vPnj/f3dt/vnMjD4vLK2nYn6bcEn9/0Aa5VvqwJv/6xz/fn9a8PvsQgz240aBMZkl4SA3ZJdUB", - "vT/91z/+6UZ16wGFGM0yQniD374xSifxd9IFClRdeO2cZCv0+8OKkQAXbAZtkMmEgBl0ZNatVw6mBgPS", - "TgrGGY6oWgQYOb4y0e7FKzWk6zbuoOpgQyKvaduiomrOJfNxmXS64TpH/2l8wzVaeNa64JXMx01+6Nf1", - "Xo0XuvRa+DEOLUIMZFFrfdnAXcznCstKQLf+O4K8C5dhtpxtY95YjbpbT4WAKBZb180LBQyhtdfkSfuR", - "v/217fT8lhWzTn3FP6w4h81H8EZW38CNHDD6RutTa2v8wV6At/tqNPZL0a2s9VepW1feujfvt0X28HKd", - "hOIGu3l/XsLkTT6sYwIDPdox2CUv2+5WSKKBmrxcmIABjSekVwTq2UQZJHPjEdRn3sLMBzI4o0s+mVSx", - "bvebsdEB9geSvVwvWCmtmXQRuXY2izqwtsH4GXb25bCjVYBhZzsddmpuq2D6ZIqvR7aDKrbLYBVYeZn+", - "XhukdDMYJzy6NFXWoHh3Hw1QSjCTKGdw+Gtete3Bau9Qt5N5e1NAgxMT4rTEtmBMYzLDcwoVKaxPZVoJ", - "xCTXVEkIGIV2DlDMDdpTpcSsnaF+zSQ3HpSThksHs4VtWDeo3+PMRbSW74KBbwKFbdlHInjXghVojv36", - "9WnXBDBA6KEZWCW+0U3UjEAzyKKLWnmF8vdw/PA4ISMYdx2uP11eRz8nHTyrgkiipMXvLsmhRgQo4jlT", - "dRz/tJ0iV00rW76ScgbBfjb8A3DZbO+GQFBMIjiRcvksVgn9FsRdyxuwKx1KHNgNkTAcCvAlhX3Fb6xD", - "uD4AY2zwqkObdvy4buMlHEnFbTmx4lSPyHVESFwH/Ay/0jZW3n4ZjJV/hS1GUFG42b4N8c7Ls+vfXYIX", - "jLVptf2YfsZZD9BJ3JZaJBEDDWixaqqEVoEe9yAtRiF41dALbTKuyfXqtf6VXCvAR4/zxIDehUnXsip7", - "Ga1b8VtnNjYdaC7I2vJ8d1C2zsSb36pwnQ1Vf4jadfatO6lXt7Q750S5d88tGTXuULXQS8Wl5QL+3SvV", - "GBtDSl1kL3i0nW7WSHBvFraKWFDeljmaDKdklAkyodcriMe8YBTjKqxJeZCKDAaDL7qR4mu09xRFMyxk", - "beyMTmcqWVQDcPYCWEqfVdRREEWYMxS22flyN92Hy9Fudjv91kPC8bkHDbRU0sSKpKNVuNlHpbfNWucz", - "vAArTqOT8Onu3mCwuzO4FXC2G9YNluuo/MSWQKy205RS531nHf2VKFW/hSLJermu7pWgkKtdLJNUguD0", - "ABJvMhwRlJAJgOQVCa3rPYv1rlcP3gpUNou2oH+3UXbfnA++WjKn6MpijrtpdJxzsYpB5D9f4xBtYDPR", - "EqReIOdutzd48nZ792D/ycH29l2AXReL1JTt8fTj9tXTZAdP9pJni6e/b8+eTnfS3aAedklNZaA2tPqL", - "frcxyqa8JKtYRhWWhjbsHDIi6gWT64XGJUkoIz1ZZEitT1NcwQuM/33t+b+Znd/MYKXscF6dpC9CYFUu", - "ToWyHgZ/y05mpe+iPpuT49WzuFUGUn0gYXqrDwXIq91goELFduczkRly1vIaeue92PoiWpkVt+4qCnnY", - "4aQHd7lhxUPkXQNm8Ga96gJfvuQCttMpF1TN0tW3RfFaASMOcdMfpYqreE99dDJlUC3d/7kIk/OVKP1x", - "p9tJPu5Vz4z9vT3yl0UgLgjQbrUvFbQII4Ni/KtXAV4pFQ9hItm1rq7H/MN2b/s5xCEkH/d+GPSeVyMO", - "uma1/OXbdm9Xfh20WUO/BKArHbX9/EYR1249V1HQLzRUwK68ly02saXxsja1uzpcwm1lg8vHS3tcQ/Jp", - "FEA/V9Kzl9vIF5pikuBFCJveM9TKmvboExkakyllso3ddndQGG7302Gnjw4tQDjosooX/fjNQw16j05o", - "mpKYahnTqP7NGQw7LW1xdV3iZrVJ3FcBaa0fFteer4dIWJdwte6a7H9GPu5nab/tNN5V6B1gV3MqKmCI", - "wYtdRCcIs1qBUsrmOKGxTaSHxEiIVztwQG0lyVoeIEs50NlJumjKFSpT6Fva23LWbBcsxk+uwd66AjPD", - "EMTOFwFEKQDE6Cr2dXKMMsHjPCrzRxMYdIn4IfIaRNsKIX99SO5d2jcgMXvCBVpv32gyaLSzTzbtd802", - "qQm2eau3B+u3+k6MIt1OnsXreZh5qR0HuxFy+5oUxICJprrsNUnQm8yHFhz9jb+CyzqvsSVHWiTKM+dg", - "0TS1TEkBdwu4GEJxvcckIfqaWm4E8SQusySoLLnoepa6/eTZrMnFCR6p5YH8QkimdRXAP4L+UswWwYG5", - "sqPFXbIxcGjf0ji8eqZckV2t6uCerpXEGrfKN+E2lVAwXL5m8zZ4KZee+bvA+PZFs2UEFMfwK0Lam+YS", - "APZLF/bWaD++C7PcQwppr63roQbb6kCFC3R0138ZC6zFuirx7oXc8yGyeGs14yYo2noWqG91Puz9j7Ey", - "o1H/YOuHv/zfvQ//GbQ21/RmSUQvJhMINLoki54pPqR19H4ViBUqH2hhempJheAUbEgAcm4Poz/e/UHB", - "NBa/4nRpChCh5VUO2l47ob/8R3N8k7eM74BPriXZzy4MchcFVBV319FGSsTUxZK7RLLN/pBBbtolWUjk", - "1SOzIo0j1O9k8YkXgY4ujBjYJ2x+gcYUCjzKIdNaLY4ikmltwpa4oaZKOQfuIwhO/HZsXTSX+G0dkiae", - "gKD3p0sovq/fvf3x9btfj0evz178engy+uXFf0OIx1XP9BD3NO3t7T+xtcn9ldwO1se4eZmHPjq1YfrW", - "1T/JQaEFnC6J0lzlEBRCrqMkl3TuHIQquX1Bh+Vk3dsXSPhMBGClklBUgkWqTuiEgF8frhMbVEOlI0Yq", - "oai7NW5QhpZvbEM4ww5wUq8mf6icht6K8GqXG1td9CezdizUYKMGDjtkvEL1/YD2QiXgVbjYD+9ltAGZ", - "I67yrEuc3bwZVuth0WAw8vALFxgaPP8SRUDfraz6OedJT6s3DZUSgtZksxbByHloymQkdJqcDtNxQIa3", - "pt0pneKAnyHkT/gixTrdgNZmTC3tf2PVsnAew3G9jIQ5lmapamUPakYCqXrNaQ6plmobgHcBWdjkrlIv", - "tq6aqJoytWWL6obwMmIOYOarspXLU+bQEXvw0fok3JV6lTczbyTNe3Pq1IeagrNigc700lzNiCDeRsAH", - "JTz/DZfM5uW0QGExRQkzIsqYVZfUo6VScDdLtFFYftwSFNnGy+bw1eUXTvF10QO4UrBc8j/CPMryT9sv", - "fwSo/Deu5CWduCZgGDXlLgwMX6WiVWviqGp5M3yqWp63eT948CyvWsH9ms5WjTjLPiqkGaLHv2GqfuIC", - "1MFmzJM7x5eHyz8mAjDg6ujxraDXaUriEc/V6vNvK+rbK78oi1qW1XWqLwYijirpvE28wKFylGNYXmm9", - "HCTKBVWLc71eNpgb0iBdLVtYSOgIfi47hvqhnz6B0XgSSBh5SRgRNILqrPo8ppiBxoTen3pF+ky9xiW8", - "VhCBXh+dWHODg/wF9ZEqID0Xd3l4dtLpduZEGJW7M+jv9gdwmDPCcEY7B53d/nZ/0AGtagZT3IKK+jZ/", - "2uYbF4rrSWwloR/dS/pLgVOi4IvfAkgAEHdoXwcVBE89JTLDVFgtMksAocAQDNVfQ7kBd6EemFu5a5a9", - "tc0U0owh+4Vkr+3mfgBBGc4OTHNnMLDA5spev5C7YxIGtv5uo0fLfltJdXaJAuj7S2qeky2Lpf/U7ewN", - "tm80plVDgbMb6vgdwzaJl4B2vn/DhbhVpyfMpOXZJGsbDuWfOCAk/6z99kHvmczTFIuFWzB/tTIumwRj", - "IhF27xo9TkkUaVYBNYL66DUj5jnCCmETuSxyBqWV3YeaQqunwLTtNrkAKfqRx4svtoSVPpyN4lOVnenj", - "8mmJnr8c7RRkvLyR9pFD2DZUew8E9CMu6oI/2EnZGzy/+06POJskNFKoVxCwjUemEkJ+EsALd9hDXKDf", - "c64wKsL5H9GRtjLruCC3bnkVbf1B40/meCckZAY/IyLFzCRHmHfWHPql42xcEuVxXnmrOcI/Oe7Ym8qB", - "8JiLCgS56hH1r626MLh8He0FEBhsn2Z68QMS/t49nHA72aI07EMeOSjIiXJJHtNxsi62cSmEBGW5l0R9", - "LTQ/uM8ryxYR+BOeosdCwC9JIeGVu7V0KWxlImdGAQ5KgG/KhEX73XdV4e9t+cSLkgG/hm4aylko41fF", - "8aKP3JoapV8tAGJJEJhnvHytnOnhfS0nbOc+ThjMuPAUfbumvl1Tq065oRY3BTiY3ilvYYO4kQXiz2d/", - "uLH14Zvtob3toZXlgZEra134Ox/3kY1IjXhMkJzxPInRmCCDd+RiTxQW/elHhEU0o3MCoHZQpC1PFM2w", - "gMiSFMVYYeNDbzRMrDRLFM1t6eZ6Lg6xXOA6joUkI8DhGzXhT5YRiJQxEiP9iYXuK+EEl8qJm7MfNLAX", - "DZZXI7qacUkKPD+mvNsc0pul0Y6h2f6QvbVAr3oBIZja8RpJEoCrXWH/4QzhIbMffO9YiAsEkzgtORcW", - "gBlIDTKl2Zbl1DY90pGMeAhr5y1hmKmezEhEJzSy07okCxvPGWywVd0lPWA3zvenRcIG2tkM47UBPGMY", - "nPe4eIYsJVX9NwyCoKMkj0snl4MQwmKMkyRYmGOa8DFORmZ9LknAJ/gS3rCLUjpcSm8S4zExJeSzhZpx", - "Zv7OxzlTufl7LPiVJGLY2ewPGSRi2LUmcbcUENEVFHJLM67PmeCp6XPLDHHrj0uy+NQfssM4pcxRBHyC", - "E8kRuYbvoL4VYGYY7tVAD+Y0hf3gR7lUPPWRTx3dmWHyXGW5shklkqhuCPVzyBRHfzhsx09bf5Q9fgJn", - "McGxphPvFTMlkK2bRi1HWM9+BK8G3O0EFmDY0RepCfOYCsyUge0swCnR1N/SjaI6AlRMra9whBnKeGYq", - "SwBRzbAmuUobgNWAkwQpOEruWy24w042zMdC76XjRtw9A5RWO0aUodMfvcM02HsWPk+SRIKEIkr+6/z1", - "rwhuZb0H5rUyXMukdDAtMKA4B9ep42kvcDRDxlEFxQSHHRoPO4U7N96EsebShsv0euBT/EEP7QfTTZfG", - "P/T7uinjrjxAv/1hWjnQZylLDQ7osPOpi7wHU6pm+bh49iG8oE3wZecVRoA2zDW3CZwEU0Ca8W58c0Vi", - "FiNub4FkgTAqOZAfuDKmDIvFqkTCwNLbFeQTE8noLcYfQ4hcHHYOhi52cdjpDjuEzeE3G+A47HwKr4D1", - "WjZXroP7rHBuFkT0ZDDYXI+Ebdc34LNs4Rj4wjpgo1ZUlN3UO2hhWP9c/oF/a/2zcP1gpjsvoYmM4u+M", - "74/QAeFJ7L4mGnBB1MRuzCKSOLF7vaHn/p0HerMikiT3TaAPRZ6Fe6xA6n9U5AibVR6jleb7B6a4wX1d", - "KhWz/cPQ76Oznwes59Z2TuYu1DlcpwQwaKwqjczLCEt0DmPqnWvl+wX82rf/dbofYCpeJHx6cWBUd5Tw", - "KUoos/kAXqCyFg/sWsJHBoam+M6i0rgicRtGkvjXP/4Jg6Js+q9//NNiu//rH/+E475l4NWgxvTFjGCh", - "xgSriwP0CyFZDyd0TtxkoAosmROxQLsDa/OHR8grdW+lNDlkQ/aGqFwwL2/C1GuTtkHrKtDzoSwn0sL4", - "6BfpxBaTMbGNAbuNO8tmKe/1RHcDcIgwA28C+lZ0NABYctQU2raaaCdsMjVzrhhN62GaS8F66/mLItfK", - "UG/PDPCGDAaWOHTu4IGdNNo4P3+x2UegbRmqgIJBoDuUzVg1ov+NJ63nSYajVBkKrLLhTRHO8Jgm1Jkc", - "G6qdmCOY4mhGGSnjiwuscdfEgRup5jGHZyfIBkJ24dUhe32+BSZWRSKVC9K1nEBYhNGyHBq3eS7QA/Av", - "qiA6rGffHbIJwZAndHJsmIAHwl3kAxYNMwDygBhXqiqV17pDZpBkLXKxPngpj0kCH0H/U6zIFV50UVHr", - "1lVHSbDSCrHs6peHzGC92jXoAVQJ8obZB35mhtRzkbw2Z0uQSaJVY4jAN2W/oe+NCRfIRjh7Vf5ddybJ", - "0gxLL1qKo9fnen5T0AS5sQdCS6/P3W5sdpHkKEooUEOE2ZBNIRDIgfdyVtnVIqFshkXci7i+BHwwp0vG", - "rxIST5t47JFPZHcoyVT6CRynn+vk+tiEi9nyBPQhNgB1qz13x/addq472+KfyXdnC0HewHlnLLjE8Buz", - "ut8ceS0ceeF1c069kGft2CEw3l3Er+nigQJ+He0tr7l54i3ZQ1j00IaDtgGvCBfo7OgE4TgWRMrNf297", - "n56podJS/tP3o2bFDxF6YsfChQX9s/aWKoE8Fnbwxo4aYTeven1d/37bqhTfabzpijo85ZV397dHrdOb", - "XCOl0FvS2rebZG2wLZURhzKDJbX0QDRKSCG+FOfUp6J1VmUTxltcOSvFJcueT47dgbw/+7LtOmf1u+Ee", - "mOJxjSE+ICOsplr7VbMfEzW/K3bRoU2vMD9/XaQ5uD8p6L5N0SEyf0zqYlxbNs0FDdBJ4wX6kigDb3KX", - "errtITDxcyLcqTYDXZhZF9MynyKD0wITAkvMat33xLzSTvU17f2ZNF9YnptILHbJv4koLZTdcq1WKbgn", - "tgT03em30MON1NsvF7ZiCSywyGBFHTu3E1hWN7BcsGjzW+TKF6doE9dYKrHCzZvEhSXboCkVetZ9yXWH", - "zK83rmU6q9dShiYJnc6sEyCmE4jVU379bhjlzj2MsqiTLbAiNkTxMeb9nulFtl7gOREKvT46MevvX6lb", - "f0DQ6npVyTGvlbfruzeveoRFPC6cJ80yqX3yhRUmQ/+VXN77P3WPMJ+VOvGgSWD8jP03weTIxL/3Kf9f", - "Oz8ldCywWPyvnZ9wklFG/tfuYYIVkWrzzohlcF833X0rMI+Y+LT+QquLBqyJTQEydo3AX7zVUuZ37/+p", - "xH4z6RsJ/sW6fpP928j+/nKtFP/tVtypAmD6eCAPV0FsodWGR98gbe7BaGop0oO0qXiRSlCbGZcKHj2+", - "/GYbVE4LivOvjZbW//JArrw+HOmeHHdhIaGiNFS0sOmD9+QLcOO4d+HW9nv/joDDdEynOc+ln5mYYhXN", - "iLRZuwmpMuDHJnaX13Oj4P0VU+ngPq+Oe5erv9H9HUn89Q01zNs49NbJ/O6ttjK/fV/L/AbR1GY227Ib", - "XVeSabMh0NphmrYl4wr063IAeGhcIV0EvdOKSqkuINAgDobsf2v94zdFcPrhB5dCmQ8GO0/gd8LmH35w", - "WZTs1JEKYUpQW0Hv8Ndj8KJOIVAWiuyVCdv1cZia3UB6rqzAv52CVDqS22tIjgq/aUitNCRvuVZrSHYv", - "7lZFqpYmuXcdydFbaMEtpvifU0v6k7tHKhqczCcTGlHCoMALJKbLpXhAo8l984zcMiGZWX+kF0xUkURa", - "q5EF11ojoZc1pe89kOykLKJ139qjK1/9OBOreGbrwVp9rZQWmhW2r40eBvd7e92/ovaYScxoRMtLl2mh", - "O1AZyNSESnOThVZ8WYKK9dHbt69cxpmW+IWrM6W4Ky7l6m4OmV9cqo9elFW7zAuuBS2Rk9hmqEIeni3X", - "FBMcJ5QRCNElMpQcVi0J96DH4ssLleF6d62Eyns+lraC6cMJlQ/GCu5FfDupFIbmpY7vl8IrTosT4eDU", - "PCp+ZRlQgPGExKctnCveszmsWzNugM3C2I5nCY4A2lG/ZlDHLGyAgRn0mwIsAMGThAiDJpflytVAHLJi", - "cJR5Nd5tjZsL3fwoZ4omF10TIQOQIBJhtrCQSkNW6QwrRdJMQWovpK3DCAXJzIhrxR/1oCnPJbwFWbh+", - "lwgnV3ghh8wmA5vPoVCuIJEBXkySPvqZAw4DwlNMmcd4TQXC7+SQXdA4ISMLo3CBqERyxoUijMQo5XMi", - "q/0SLBJKBEziCOuVkyjFC8AzM9COZn14RgxmWAWsget/YxZTqGWney6mfDBkGO0MBiglmEmbei3xBC4c", - "2waCQVQG9D3CaG/w3H5V2zfA3HXLv6FPkxBkziM8ThaIaCoG8Ae1CRuY2tqSpkav3r4JFdLsV2EytEXD", - "KhtLpSuVGHdRzsrkcjCf56zIBdfbpXLBYJ7WsUaoKK5Bi6cxJhHW68l4tR9AMuRRlIvQBam32ity+u8o", - "OHrTO4elCqduJ6CFRySGPWdczeBMczhKm983UFVJVH+OiyZ4SLhAGHl0XRoJSJQDa9wA5L+LsmIfcxV4", - "Lza/d2dHH1/LCNzxN9h7j+V+AiLik0nlAK6/mswBXpUysUzCf9ZzeuRKtfosLqZ4yrhUNHLMsF7Z/ZtC", - "2FohXL2yQWqecHHpy1ZV+v2Ji8u2GpjFE6WPSxHzZ/gV2vb18AC7+eFN/GBgNsqKJpp7V9Lq9FWcUhC6", - "qJIudpijhLOpPkWlofveLfG+VrdhcNj0ZSqM/7hAzdFKyMj+aKq96snYWppgtY9sqw/Ni3Tv9+Df+ZUr", - "RNMsISmBarA9Q2x6s0uEJaicT6WHM3QzXqlPlZ8ObHRBaVz6XScOAV25DdsA6X15u4JMNeHT9Th+RecO", - "tC4A5DdkprI/QRfGm3OBCh6sBVqDmo+uZjSaAagf6K26fYP5h7PsosAz3jxAL+Eg+7DO0PmGwcrXtCZ5", - "QgxW3zxNLw6W652+Pz2Fjwyen6lsenGAXI3T4v6Q+i0fpE/PIsFSoV8t9OBGoYzDjl4orPXNYn6bFr6v", - "xJseshCUHyNXtkE6QRceqt9FA+SU47ev+PTBhLFuc5UAMxfFkVUdgTYJiztNcRM0CQP6bQ8GIfDqluCC", - "Zhh3jC24NJhXfFpUKKiQMs6ytuRrhwlUPE/TFTSMNjykMqlinqu/SBUTIeBjS91NxI02cGSrU+FLTagW", - "l84d7E0gv2B0kIEMDy6VZqqdboewPO0c/Gb/NU/TTrdjx+NBjd9AuF8D0lhvcDmKRe+Mh8T4TSy/CcZi", - "ldl7IIu1m8Oq080S+Rvzwp/eW+hsdg9IhiAf1Iy4X5MI6o23avBhvACLhJE9v4+RAaIkihIuScXB83jw", - "qKyhqyYzNhuK3Br39PDi3BXwaRMUcm4/PXdffgW697pYETdm5KZ770EjyyN4zLm1cmk2Ey7qIEbrokm+", - "ekL6cluyNNU2FPKNNm9uZWxFmFpPWGYR9oPYFHTDueIpVjSCYkLRjHPpkX2BOGzKflnjcUGZYFoxWq4N", - "yr/QpHphzdAXVo04sCYzhP1Hto8+fG5D+cNfuEflFz95VoGC43ed6A+A+1DtXFAyQRnOJdFSXZ4SFC0i", - "zRVN9SiCoxmKcKZyQaAwHkEpZTTNUx9KWu/YHAPsxcV2etFF41yhBIspaGXmoQu2iXiaEhYTsM8N2Yzg", - "OdUqpUAJVoRFi54kUFB3TtAVF5cJxzGYGLIYg6cHCvIJoikQcLlTonCMFQZB50Kf+JHJC7ooauwatZ6R", - "65Ia4iETOfveFAnQzV64gV4gAijYVM6KWowRjgmLgujQ5183G/vytuhzouoTfaDIoFvx0ocMFfJtrm44", - "X0cU0SMLb+bCbmMbNr9C6JXNKmw1ocKR0b/nkTZzdXN8IAdTscSrTvHX4VkqiO6r8S49vPuICxTnpjvv", - "VAKZ/1l9QgVD8YOtIFnTbONtHUNF0blimW/E87b+cH+e3MKW95Vwwm6jYt9U3qic9NfAcu2q3ornPpAR", - "09qSfJvcw7FgF9H1YOITFx6XeyzGVsuwzdEs+LbPnZTAoH1x9o1t19m2DXi4Ldt2ttkll77HyCnrQYxo", - "mINbM24jq7amg3/TbJTa7DyW+eAssvRc3BtbPCkYoWGNGV4kHMd/hiDhFf6jiAthECUAo+IxIZp6VkM/", - "PQBsc2XdtK7L1nx/errZxCWEWskjhHrEHMJLydGfpfGyAff1nAhBYwv8iY5Oj224LpVI5KyPXqdUIcXR", - "JSFZmdECWYV9PT+HrbFco70CotHtEKbEIuOUqbWjKF+9m8F8ulVl93vmkxYi+5s7vLU7HCz7j4+dAZeB", - "nA0zgdWaqcJqbelOyiZcpEYuw2Oe69Y1D9LLpPfTlMCc0ITIhVQkNVGJkzyB4wblFmxJXfud2eUuxOTq", - "k2PS5TIiUiol5UwOmc0VyYjQfevPdftegFXQIaBwwV/PDJP8OoL39GBMvBpWTasGKEhQqrNz0NnCWbYV", - "Y4UbAsTs8D5jSD9BNB6Si3TMExqhhLJLiTYSemnUEzSXKNF/bK4M5xvBd1+6YPDtT5Ze6RM24cFybIZm", - "C2L+U2V1WbbmHJOPjq29JP5hcfwHNjrM1taXJBYEJz0o8euwcFCuaEI/GlanG6FS0cikHOFi7d6fFky1", - "P2SnRAn9DobUtiQxiAagXW5lgkdbw3ww2I0yCoBquwQGBwyv+XEKPR6dvTNpqCTlYtEdMv0PaPjt4Znx", - "7k6wtSZ4A7W1iNHJ1us1Ac7nsEz/xhGCZoIr0QuCG/7NJXhzjJHGMyQbjijPVqlKPPvTh7BaCe6bXeFx", - "2hUA5KmYzcZU4AiEYjnLVcyvWNiGMOdJnup/mD9O1kGFKRzN3sOrX420a4azths3wUdxKO2cYmLKRT6I", - "08Ms2GONWdUL56YAQkwlGjB4CxyqPyN1f3nzvb+OX6G7066oK8X61Zyt+7757Bgcwoa/Ho/lmBtKczNR", - "fLX16QrTZuvTjwmPLqWFYvHNhlpvA8hy/WMJMW1dhCAmQGYoshBGBiiLyO6Q1QyQBvFHIowUESllONmC", - "OZtGACzbWbHwnFNI0I4gT6UnaQyYSQkgYgP8nZ4NGKpcA55HV9piVf47vjNScTQmEU+JAxDfDKluf8NU", - "/cRFFQ38a+GLb731B0hATMHevgYAvbnHzwJEP8XXECod59ah7Ea08ZKXPxpTUBfB3gw7uwM57HTRsLOT", - "Djt6B44wmFCxQvsopSxXRPbRsbFvQQrukwGSJOIslg7H3FnwdgeyKSHXkGVDducT+O4+xR5LVbCUb2wn", - "Ifag30P6e0jaQRv+gbNnMu7CoYsRz5Ux99tzZd+KiQLzyOa9+2q9M/JNt2/Dyf9mj2+FR8Eua3bpbb3h", - "7FkuZ6TZ5PbK1AbK1RjwsV29TjlDf+dj2UWMXBlruJCqv8T39NdnpoP7wO7XXd0Et9/O/RtofwvQ/nKt", - "wmCNJsBSX8mOOgxiI7nOuFCA4mhz7Q0NgSYByBE8wgl6fXQyZJFmRQZaUJCUA3eyEOPmFj782zl6cfSm", - "i46hdiT6OR9v9tFrlixcBW/joxkyI4kZ5hVhhsaGakkcup7N2IF67jJYXHfwQMWYzckIeFbcXrkg8W5n", - "RnAMEskfnVfcdBZAHX7zSh8gAP41Xxbb3lkpfHTeECUWvcOJImK52VObJ8UKzAx7STsIOiu4GeBL3aF0", - "yGtln0Y2MNAYuzudAFLGp291FO6+5uj9eMlMnIipYDfOAWmUQZIBjhePK5ZJzlDBHEMs0L+ui0oETVnC", - "lpetVDCgy6bI76/I5L6Sd1Ww5f9dTxfM9NE6mrLKPmkiLiqYrPX0uuTgmYFDto6qCGc4omrRRThJ7B1l", - "b4IiIqVXiL9jQfBlzK9Yf8jeFLVTbEIvOjp713WOWhRTeWlasL7YPno9J0Lm42JwCA6a8RrDmpN4yBRH", - "EU6iPNHiBplMSAS5uFASRTb4couhdO7w7JSdBOu3eFHt+aMrGxemCdi9kizqFLdltnpLkCjBNG0GH7eC", - "GgQcQqjBWDfKGaJsktiQqkhwKZFtqkcSOqXjxAYIyT56OyNI4pQMWZZgxohAuTRR8XrovUwQKXOT4K0b", - "AJBeQ1FdVAILZoIrG5qQcC6kiSbQFP7+FElFshVk9sa0fApzviPZ1jRue3ogI3VtDM2mEPsK0htiKMUs", - "uKajPHEBjPcaim4G9NBS4mM5+G8FnU6J0KcCGyZrwvHMsXbLaQ59JWO5sYTkefFWuxKSRateVqKXsbcS", - "GG5UYm3HnZtF/QU6v6SN2IH20c2yiH/RH7Xsu5qtGh6EffSZs/yzVOY/95IE2xqwSgp/bOYkb+SVo1pJ", - "tF0Pq9U6s/YuM11b42c9GGzWY0bLwpX02SaF9+sjhMH9ojzcd5G1x01bFbSrim7akPK/Hk3/q6DAu4HR", - "f2CUk1vA6H9VefeAc/5w+CfBg/pQefQV37OrX/unR8K/q/R5A4cPcGxN6fOG69ng1ZWK0nv7Tjs1ybb4", - "Z5LgbbzjDeR3t+zftP4WKoO3WOtc0JrgSZqphQtos77KMuhM0o+k3+AILuJW784VfIuQzi9HHo5OGwM6", - "/5zl5h8kZtSWDqQSnRwH6rg/MoxB/8xVLpYtfev0sIhmdE6aje7VE2yXKBOkl/EMnCuxWTC7Hu4uU1j0", - "px+Rbd5irtp/Qe1JgOonMYqpIJFKFqYOqOYIpo/vJBJcawLwnItFc5SIOSI/CZ4e2tmsuQ/tmbLGsDLO", - "MF30Yqxwb+64zQoT2mdEd7p4Ss3wEGXo5Y9og1wrYSpcoInWfBCdFEtqCvhLoMlNf8DbgwbLJv1IRtNx", - "m1GuqFXy2taCQVEuFU/d3p8cow2ofTYlTO+FFvUnIMlmgs9pTOLKGDtznphV3W5Y0JvaXbVQURSuc8qF", - "GdyDyDBtLqTpR5pV2UIREjOmDMPg1lYFqZ4pk8Sv+8OUuQAcu0duFN+uMKv5bThlR1Mi1OG0i6g4NxDP", - "m9+uucd8zfnJUO5Oq9x2LjxntfG6XX5Uy7Sluyj8UOTO3a/Z+v3Xk9JD5aPM5rGm83mhkDaZzb8uEhzc", - "3/1w3+by9484BfQlccq3ZyqHBnSLIYJ5BTHdMZmThGcp1EOHdzvdTi6SzkFnplR2sLUFsd8zLtXB3vOn", - "u51PHz79/wEAAP//zu89IfTvAQA=", + "LWTAlvvo16Kwll5qWaIq9QOWwho9mQBq/cJIJjwUjfKTIATBs7Kgpz574ME2GL1F1ACOAaOJM1dWEG1A", + "lFdMJPo9xwIzRRmJ0fuf5PdoYF3m739CJovGMiAq/cDGqqb4NLSljdlwZ7OFpBFOzLKY0h6U+VZKONKt", + "1Yyz8kNrzw0oG+Gico59oI35NMthAc/f9E5ev99KYzLvVsYE0YgznhA97k2Px84dJEOZuldhrfMmc5Gh", + "btmW7XhrVfC91ovkcbnA6nhU0ERw73+qEou5eEyqqjXXUkYVQG8B3vJGMUiDEN1Hp3hhtfoMOLjpykNA", + "xZOJAaAsOJsgCcESqlFL9P6n/towQcUVTprm8FY/tKdmQ09I76keZhdlFaKEk+QhP/ndPgkysHI+AacH", + "dFi3oPfRCQsfQhOLAthtBoZJSjplWtOQNhwlwqxYyaW9q3LyoJGijnXXqS5cZTrdADsKUUyId5rcumWh", + "e7k6Jr+sHhp+ub4io2mkud8jl/5X87JYzcbxN8gSdHExyHxq3CjWVidJhgVWJFnUAnSrCPtkOcSBXJPo", + "BvmHL/Trn0xVlVyQkZoJImc8qQbG7HaXK/NKCEKfE1uMzMzJ8wQpjlIsLuGUOc0O5cysQDWHYXcd4NBM", + "qewGk/r57dszY+5RRMxxUs+CkUshH8ckwQs0JuqKEOamgiXCfhB0PYtYNhSEEmqUEUF5dQ07u4F+z01g", + "PJoKHBFkvnJ1vQu2po9e26W0vQQgLKOISNmwv9ur9td+OsmTdnscGtb22jL40U02+O3RmStsVBSOdsu8", + "s7zKZ0T0zJFzFaRXb+2OXF1iy3XFDDLxkgQ5JlBzy6Ye+ZnBLuANaorpzyvZuB5zkH7+qO0HToFZqq45", + "5x9aqRv14x6K7Egxi0MFuE3GicFqmAKAG4S9ixx0ARqbaDhzJ3v3s02cEQTHlBEpa4nuUS6STrfTm9hZ", + "HWxtaX6fAIbl3u72s63VccUrA8pt/NwopqsMDi7KzsRhubRqAwAIk66SxBbOshYmWbOOa+4HYE/LAaxQ", + "dFvfbZ7I7zIaBoOlnP1rHClXJhBstBUfPPaPLSBNVwUZ3WBqKv9o3eu5fz4HwaCIDKtZlfy3lmgfAqUg", + "yFXTiLF/1dbREPnHoHTKRagWHBfKpoCOiYuCLe5DF2PqMJUr3trBM3+WT/b3d/fX8SFgNrVjbs9dYKrm", + "7WqSO/GW255f2wDE61VlDnekV5ez1euyhqY0R1wiqdULyjPCbrSe+3u7Ozdbz7YTOXFxgjW+FMIIOjo9", + "NjKR1iwxZUSglCgcY4WrTAaMm5rLQLEhTFLIE5t8v5q1NATU+KA/t62U9qXCMRqKJr5xGOEpZnQCSpJ5", + "0+9ZzvDO/pMDU9o1JpO9/Sf9fv+mUCgvSuyTVluxZaI2PVSUvpx93j7cAeJJm7n80Tk7fPuzZmS5FObS", + "2pJjyg68fxf/LB/AH+afY8rCSCltqgHTyVIV4GqAYm6hqEl8gMqC707uaRMw1uCdgHB1gGcK4g5Wwnbv", + "DmCwoHFarSNxg2TxFcnTWlx5zZJF49rcurBvWfJeeQV9fbtCi+K+9OPqgAtn/4R3bJ/G0lHUPV4OtbhV", + "5Wq5srjnUu23jLCinGeSmL8izgDpOVTbs3JFumctSsPBNWJrwBVd+j8WvXs/HvkD8X53peW8n2yRzw83", + "jJFaKZD+bVkOXc+FnDi65jCHjdHFrdC2oLIFJgwmBzzwXXibOMJq76+n//X7/5FnT/++/fur9+//e/7y", + "v45/pf/9Pjl7/VmgN6sxKR8UWPKLYUlC8FwFULItKZ1iFQVsdFr9a1hh+8RYHFQ0gxKwaEwOhqyHXlFF", + "hCkoWMuGHXbQBgFNCb7S4i5USzKJiJv64zPj4tYf/+HE4E/1NmKLUiDshhTgMzIfxzzFlG0O2ZDZtpCb", + "iAS9QP8VowhnpiogZUjrvws0FlDC0focy8676A+cZZ82hwyssuTawKtnGKrgTYo0QOYiBuyoTFy0fZ0U", + "OCQmRXXIitu6AGU0jud+WcGBkqSeRNawKKv1N6s5PRuE4CshwUlvJFQpAhWkoGxNRkXmFXo22FzW59bo", + "GAUNrSA/G5lhMmAP85C5uClr9pjENAK+4hJHZza1uMjZNZRmjXiZ4NcL2Js3JpsxRjhXM82LIou0EHF+", + "SUkXtrQL/lEIBYIvTYDHjGe98aI341mBuoGFCX/CJkSiqmT/n56daO89EXRiewqCJ2gSCQidcGTszEwO", + "amFdWJrYW1PghWnRZ07s66ZSkDTFYUxsv8oFc/VLCFQxBXiXgvpISCb/HkUJBauTnPE8idEMEBiVbiYE", + "otgZFDlUeBzFZFL/dzXGZWf/CWiw7t+7O61TmM3SraKyPAnotKljfS04tmGTMAAjHoycIXxNVJq+Aa1j", + "H+wUisN/z5FrqDxxBSMxnj6TVCltOYxEeumUm8G8N3sMLIbMCNvT1OY+WjqFlTS6Fi2YcBX4LGmBXvPC", + "pO++fXWOFBGpA1TYiPTuwCkxUCU9KmVuK7MdHp2+2Ox3gshbFZcWbNXKNLvqoAPgGzZ8pSkqp7TR4JR0", + "0ckxpE/ba6XUxSDf5ScuUGJuxfIyOgD0lqq5B5uajifHVgBNFmUMjBFbhp1N12JWv94O0JtCBcTFUIpE", + "2JK2XJPlZQLN2ohIk4yz1Hotbxr8Y1b9s/cxpN5AwUvDiwE9tPH+am9zdFhk+qKqWchufCH5YUmN9i9v", + "7780zvaXl9F3byajWy/0KJthGaLume/VhJeW9t13ZFfZvWgOIqv0O5I0eLb+5sr+eNeQIvqeq3weApbd", + "721vv93eu7n57qYQyVVsNA83sUBJbg9vfBcwwQHQX6pGjdkGSD+2uQXOLvL+FM2wZN8peFizjmzvPm1j", + "lIBe28bp+xH6fGKGVHApB7RWxJcbyLlLmiRGgJF0ynCCnqON85OXv5y8erWJeuj169P6Vqz6Irg/t0BL", + "hlsA1tGkHAawtioYEahIJn379hUcroRAPo6Rwy9vj6G81rTYAlPZDe7l2Ttw/GM5cpG8zcmruEwAJ9dU", + "KrkMs9cqIP5zMJzNp6VhrM0kTRs2xG8tEPTPFaTiIG7i5h0gOLtshqXlfABw44fMGv36gJVXQiF/Lp6x", + "tTPcEZxx45UWggKuQWTsN91utwcmvpPhVBCGQmzLl3BcSv+tkYC7HRpIZz60cXzo5Kws/VU6I1zztTk9", + "3+lvP3kG1Wu3B20Ye4qjFX2fHh6173ywY26ZAzw+iOIDUNhv67OyhG1UEJxc4QVUfzRLO+yYC9PTbr1j", + "axXJVvE1y4DLt8NXrotxDQjKIM66wCU5SlcWn2mRr1pH0UtzixGa0iShkkScxbIqI8+wRDIz0LimCEsh", + "wQ8ZDLCLilrYIKUgHEUiL02PVrq28n6eWbqHQrAZZ1oHgEoQv5CFRCkFJ2jRPYQ+SlSkRcVDtiFcCl2R", + "Kwc1YGP9AySkdG2qQ9yFqGEoNqM/GDI5y5VmYpt9dMSZzFMirFUWjSl4jDaRzI1KC+OF1VhohilpTMSQ", + "6dcC4Lt/FOrJwZPBYDDodgpNblf/exCipjt1fvYtuLRJAgf4R2ZhpgFXUuQM5SwmoigITww51EPkbug4", + "/UxUafd5O/HKfl7KVeGDuQ6Euh269OdC+sJQG/RziA69hXK+f3sRvVUSmpNfbfqZ/Wp0kwgGgiKeJ7HW", + "+Mb6tjMGORJbM6QkynDnIhPknSnGWp26DT1WHP2eE7FA709PK2EPgkw0D2g3ceASDfvAsxttw84aG8na", + "0dzEvewBIN8H6HFdUvEkxC8Ocex7GF1WvKHQimGrojivsq9prTKYkUOZ2SdNNCsmWKvHazBEysBIv1W5", + "kBML+GlFaZdTYQHICzyQ0qkvF3JrnMutLKJbNpdpC8BangFYy14wmz4m81Geh1Qj/cih87x7d3KMNuAX", + "ABs2CTKV7jF+sv1s8Ox579l4+0lvLx5s9/D27pPezj4eTHajp7vbO7srkopapFfePmMyqDEH4piLqPWR", + "i54PBTU35S7UZBMbj31FWcyvKtdfMEDW790G367rfjm0vvUQgilBCZbKmC8aONkpXPIk0m2bgHSbwlvU", + "3gobOp+8HWx/rvUHBtdwR7wVOTNuVQM6UbgQUm/A/mZVx3k7lg8Dcokv61bL77z9og0O9p8f7H/uornk", + "jXVjrJPTPW5uU0SYA+WuZYe4lFXPjuQMlB0rExmrvk0m6XQ7Rb4L/A3CQC2WunjcKomr6cB2w2xk1bXS", + "kE1/UtFXIFLFgDLGB1pScfoIlPgoMBu0CHSU8DxGni3OYNSBH+7E0110M+AWsyY6k0ZrkjEgM5JKm89H", + "mWbE4H/UjdjU+wP0Et6FRzg1ap0dhClo47vecLww8TL6fLmujZK1esjnVr+Cb7SyhfS/YNp6GazJdnUT", + "Rjo7QL9y+KbQ9hiv237N66BmLb9etxNvWPx2B6UCnVlR8wD9VIiXhYBqBdINSeyfI8uwSgSjzQqOhN3x", + "jqaWcuc8TIRux6xop9txCwXYCcsoCu9Kql86fz4phgLJCE7gLJdJ47miicVth5lQqWgkbfKI3twmscfm", + "ZpJ4ZJSnpphUk/pqFaziIydVvT9FGwDN+RdkDdv6X5tF/Grlrtt5vvf8ydOd509aAXCVA1wvGh9Bnvzy", + "4NbKyVGWj6xtpGnqR2fvjO0jMlaFIvbl/amPd5IJrlmPnrlr0O/8ef+5jzsW83yceI5FC1JoYI5hw4LQ", + "egUvaoiD/J0mczqZsN8/Rpc7fxc03b5+InfG2w14yqajsNntxA8uWLJRk3HP1M0KQ0MBQQnZiJ72hkiY", + "ATonCgH99BCOQL0p8qktyTmMNbviQcLa293dffZ0f6cVXdnReQdnBEa4wKVsR+AdMXgTbbw5P0dbHsGZ", + "Nh3ICEDeM6v6hs8ZskWvB1WBtL892A1RScPFXVKNbXueNi75e6s+2knZRYfk7UK1XDrlwdXe3R083dt/", + "tt/uGFvz8Ehcr+YwLmXJLI+tzODv/AZIk28PzxAkBE9wVLXtuAixG41K3WhUUFXEVAO4wcCePX2yv7e7", + "s90OBjAUdGIBLisHtsq7AocuQBSB3QgsxTLr7TbdFiFxyhDYGxIlmKaHkUuxqN0+BvV/JMxr5Sa0uRis", + "Br50cbX4tpVxqzBZmQQdIxpwgXJW1Jrpr3fJfhHPajPXNtfDeq4eSsthevUsXpWpqXeLpcwEmVOeyy/Q", + "EFcmZ3aScC5u9G2TwvKGyDxRxmZDJXp/+h3wFE1rSCqSVXUoS40rUL1uObkbnecKiYSJvGmxWu1Gm61f", + "NeFuw6ntrgLUqHCDRiy9WHOunK0P/jzCSZRDdSVc7KeeFYDCASRAliULE9ufJJwzFM0wAyeJ8CCw0Iwn", + "cT8YCaufjCbBqAp+hRJuUMAvCcls4SEzCP2ZFmHonKANv+SeIaVaIdz91DAZW1qmSo37abiiJ5ahZLUi", + "FV6vJ1bcA6g2n1QsoQmfSlAKFWQt9Ot1ETIsTDICZqaQ1jw1umQg4DowxBozD92o5iblE6vgWpEDEs3N", + "SuJIcOmBU70/reUvr8h5K7KY1wd0VgfbgnSNQzNwlRl4tNb19kL3YyCf53NuSKBhyBlcESrpjJMpZjmU", + "IvII2Rri+63DIWdcqlEB8nXDwUo1gvoiuSAlfGKRdV/Yg9w7wXvRsbbbLJeNO77V10tUFW6qaYDNPDW4", + "ouHV6hY0GCLjZZSzlcBqJdxcHczqJuiFZUEKKqFV6uHYoQ3IefHYkodJuNkmSCassup+lrRVWy721d7g", + "vC3O32pYvzOsZidswgPYHzfwnDpLtI1WzYhwUHsxYZTETpcsXKjW1AUJ44kkKM6JXTkjnwpsFxyb4w0+", + "K+ZsZJRNa7y+3mEb87AZw+ryI9CvfbFNuJMMJ9S+FTmslYlXlAiXqbWtgkCpHIXdWcsNCzLNEyyQRehs", + "M2S5SBPKLtu0LhfpmCc0QvqDul98wpOEX430I/kDzGWz1ez0B6OmWlXnZnA2L9BsSK3fcgo/6Flu1rKS", + "wRKzZb7fAsdom+ixYKT4TzQhFinxHaPXHqFX8fn3dgZN2fINjVby5JehQm/KuS3JBk98LgO5hSulHFdl", + "i8S2aIIRe7JcmoI/LW4lh8rrXIC38+hUE0c+D5rkyPDrGjAJGhPI+3FTW+YaLdhim6kEa43kcob+zsdV", + "g2jbsN9ABbsNVkJkCDIJxvfDjq40SJs3ltbE292bYFAAW9UThY9uCO2wrtZfGV/VxE/eLJW9mxG7ZNTN", + "0ZTAa1HSxcV/FPAFttf2OAb1AoWBeGVAqZFqAaV+ob7Swqu6KdGYCwGQ5FrC4czNBmBXtMyj19rhXqG3", + "M7JAgqSYsiGjrDCSApgaQYzMifCyZLnQStaUxH30N0/FAxD3NFMLWx0AjOffScSvWDHGIfMHqRvPpW7n", + "kBnLosgzVakfqpsFrU8TCmQtgxNMCSioSdUMTQSRM3/uoSKqWsa74iJurE61QO4VKHoEPlak+CVhPisr", + "mgmqhqahkflqOYrPVECGp1b/RJWixKhedHh1f7kkIiwkFlMqXmkVuuIdFU85MSAwgIgCxSbtX4bFFygo", + "LTBPyub/6posfzorGq/+VnvNwzVxuNOHxmwbNMFGJo2nFuxT9aStDVWBNLhVaDbLvgS04UKoXcWeqiTg", + "Vc5pdU+2y8SrJwu40WxJElV733u2//RJy9JFn+WsM+hdX9o1N09XuOQaduq0jd/n2f6z58939/af79zI", + "w+LyShr2pym3xN8ftEGulT6syb/+8c/3pzWvzz7EYA9uNCiTWRIeUkN2SXVA70//9Y9/ulHdekAhRrMM", + "Gd/gt2+M0kn8nXSBAlUXXjsn2Qr9/rBiJMAFm0EbBKC46ZyMzLr1ysHUYEDaScE4wxFViwAjx1cm2r14", + "pYa13cYdVB1sSOQ1bVtUVM25ZD4uk043XOfoP41vuEYLz1pXQJP5uMkP/breq/FCl14LP8ahRYiBLIrv", + "Lxu4i/lcYVkJ6NZ/R5B34TLMlrNtzBurUXfrqRAQxWIL/XmhgCHk+5o8aT/yt7+2nZ7fsmLWqa/4hxXn", + "sPkI3sjqG7iRA0bfaH1qbY0/2Avwdl+Nxn5twpXFHyuFDMtb9+b9tsgeXi6cUdxgN+/PS5i8yYd1TGCg", + "RzsGu+Rl290KSTRQk5cLEzCg8YT0vOIFpvCTzI1HUJ95CzMfyOCMLvlkUsW63W/GRgfYH0j2cr1gpbRm", + "0kXk2tks6sDaBuNn2NmXw45WAYad7XTYqbmtgumTKb4e2Q6q2C6DVWDlZfp7bZDSzWCc8OjSlN2Dau59", + "NEApwUyinMHhr3nVtgervUPdTubtTQENTkyI0xLbgjGNyQzPKVT3sD6VaSUQk1xTJSFgFNo5QDE3aE+V", + "msN2hvo1k9x4UE4aLh3MFrZh3aB+jzMX0Vq+Cwa+CVQ6Zh+J4F0LVqA59uvXp10TwAChh2ZglfhGN1Ez", + "As0giy5q5RXK38Pxw+OEjGDcdbj+dHkd/Zx08KwKIomSFr+7JIcaEaCI50zVcfzTdopcNa1s+UrKGQT7", + "2fAPwGWzvRsCQTGJ4ETK5bNYJfRbEHctb8CudChxYDdEwnAowJcU9hW/sQ7h+gCMscErF27a8eO6jZdw", + "JBW39eWKUz0i1xEhcR3wM/xK21h5+2UwVv4VthhBRSVv+zbEOy/Prn93CV4w1qbV9mP6GWc9QCdxW2qR", + "RAw0oMWqqRJaBXrcg7QYheBVQy+0ybgm16vX+ldyrQAfPc4TA3oXJl3LquxltG7Fb53Z2HSguSBr6zXe", + "QR1DE29+q0qGNlT9IYoZ2rfupIDh0u6cE+XePbdk1LhD1UIvFZeWC/h3r1RjbAwpdZG94NF2ulkjwb1Z", + "2CpiQXlb5mgynJJRJsiEXq8gHvOCUYyrsCblQSoyGAy+6EaKr9HeUxTNsJC1sTM6nalkUQ3A2QtgKX1W", + "lU9BFGHOUNhm58vddB8uR7vZ7fRbDwnH5x400FJJEyuSjlbhZh+V3jZrnc/wAqw4jU7Cp7t7g8HuzuBW", + "wNluWDdYrqPyE1sTs9pOU0qd95119FeiVP0WiiTr5ULLV4JCrnaxTFIJgtMDSLzJcERQQiYAklcktK73", + "LNa7Xj14K1DZLNqC/t1G2X1zPvhqyZyiK4s57qbRcc7FKgaR/3yNQ7SBzURLkHqBnLvd3uDJ2+3dg/0n", + "B9vbdwF2XSxSU7bH04/bV0+THTzZS54tnv6+PXs63Ul3g3rYJTWVgdrQ6i/63cYom/KSrGIZVVga2rBz", + "yIioV9CuV56XJKGM9GSRIbU+TXEFLzD+97Xn/2Z2fjODlbLDeXWSvgiBVbk4Fcp6GPwtO5mVvov6bE6O", + "V8/iVhlI9YGE6a0+FCCvdoOBChXbnc9EZshZy2vonfdi64toZVbcuqso5GGHkx7c5YYVD5F3DZjBm/Wq", + "C3z5kgvYTqdcUDVLV98WxWsFjDjETX+UKq7iPfXRyZRB+Xz/5yJMzlei9Medbif5uFc9M/b39shfFoG4", + "IEC71b5U0CKMLCFzkqxeBXilVDyEiWTXuroe8w/bve3nEIeQfNz7YdB7Xo046JrV8pdv271d+XXQZg39", + "EoCudNT28xtFXLv1XEVBv9BQAbvyXrbYxJbGy2Ll7upwCbeVDS4fL+1xDcmnUQD9XEnPXm4jX2iKSYIX", + "IWx6z1Ara9qjT2RoTKaUyTZ2291BYbjdT4edPjq0AOGgyype9OM3r2nFpxOapiSmWsY0qn9zBsNOS1tc", + "XZe4WW0S91VAWuuHxbXn6yES1iVcrbsm+5+Rj/tZ2m87jXcVegfY1ZyKChhi8GIX0QnCrFaglLI5Tmhs", + "E+khMRLi1Q4cUFtJspYHyFIOdHaSLppyhcoU+pb2tpw12wWL8ZNrsLeuwMwwBLHzRQBRCgAxuop9nRyj", + "TPA4j8r80QQGXSJ+iLwG0bZCyF8fknuX9g1IzJ5wgdbbN5oMGu3sk037XbNNaoJt3urtwfqtvhOjSLeT", + "Z/F6HmZeasfBboTcviYFMWCiqS57TRL0JvOhBUd/46/gss5rbMmRFonyzDlYNE0tU1LA3QIuhlBc7zFJ", + "iL6mlhtBPInLLAkqSy66nqVuP3k2a3JxgkdqeSC/EJJpXQXwj6C/FLNFcGCu7Ghxl2wMHNq3NA6vnilX", + "ZFerOrinayWxxq3yTbhNJRQMl6/ZvA1eyqVn/i4wvn3RbBkBxTH8ipD2prkEgP3Shb012o/vwiz3kELa", + "a+t6qMG2OlDhAh3d9V/GAmuxrkq8eyH3fIgs3lrNuAmKtp4F6ludD3v/Y6zMaNQ/2PrhL/9378N/Bq3N", + "Nb1ZEtGLyQQCjS7JomeKD2kdvV8FYoXKB1qYnlpSITgFGxKAnNvD6I93f1AwjcWvOF2aAkRoeZWDttdO", + "6C//0Rzf5C3jO+CTa0n2swuD3EUBVcXddbSREjF1seQukWyzP2SQm3ZJFhJ59cisSOMI9TtZfOJFoKML", + "Iwb2CZtfoDGFAo9yyLRWi6OIZFqbsCVuqKlSzoH7CIITvx1bF80lfluHpIknIOj96RKK7+t3b398/e7X", + "49Hrsxe/Hp6Mfnnx3xDicdUzPcQ9TXt7+09sbXJ/JbeD9TFuXuahj05tmL519U9yUGgBp0uiNFc5BIWQ", + "6yjJJZ07B6FKbl/QYTlZ9/YFEj4TAVipJBSVYJGqEzoh4NeH68QG1VDpiJFKKOpujRuUoeUb2xDOsAOc", + "1KvJHyqnobcivNrlxlYX/cmsHQs12KiBww4Zr1B9P6C9UAl4FS72w3sZbUDmiKs86xJnN2+G1XpYNBiM", + "PPzCBYYGz79EEdB3K6t+znnS0+pNQ6WEoDXZrEUwch6aMhkJnSanw3QckOGtaXdKpzjgZwj5E75IsU43", + "oLUZU0v731i1LJzHcFwvI2GOpVmqWtmDmpFAql5zmkOqpdoG4F1AFja5q9SLrasmqqZMbdmiuiG8jJgD", + "mPmqbOXylDl0xB58tD4Jd6Ve5c3MG0nz3pw69aGm4KxYoDO9NFczIoi3EfBBCc9/wyWzeTktUFhMUcKM", + "iDJm1SX1aKkU3M0SbRSWH7cERbbxsjl8dfmFU3xd9ACuFCyX/I8wj7L80/bLHwEq/40reUknrgkYRk25", + "CwPDV6lo1Zo4qlreDJ+qludt3g8ePMurVnC/prNVI86yjwpphujxb5iqn7gAdbAZ8+TO8eXh8o+JAAy4", + "Onp8K+h1mpJ4xHO1+vzbivr2yi/KopZldZ3qi4GIo0o6bxMvcKgc5RiWV1ovB4lyQdXiXK+XDeaGNEhX", + "yxYWEjqCn8uOoX7op09gNJ4EEkZeEkYEjaA6qz6PKWagMaH3p16RPlOvcQmvFUSg10cn1tzgIH9BfaQK", + "SM/FXR6enXS6nTkRRuXuDPq7/QEc5owwnNHOQWe3v90fdECrmsEUt6Civs2ftvnGheJ6EltJ6Ef3kv5S", + "4JQo+OK3ABIAxB3a10EFwVNPicwwFVaLzBJAKDAEQ/XXUG7AXagH5lbummVvbTOFNGPIfiHZa7u5H0BQ", + "hrMD09wZDCywubLXL+TumISBrb/b6NGy31ZSnV2iAPr+kprnZMti6T91O3uD7RuNadVQ4OyGOn7HsE3i", + "JaCd799wIW7V6QkzaXk2ydqGQ/knDgjJP2u/fdB7JvM0xWLhFsxfrYzLJsGYSITdu0aPUxJFmlVAjaA+", + "es2IeY6wQthELoucQWll96Gm0OopMG27TS5Ain7k8eKLLWGlD2ej+FRlZ/q4fFqi5y9HOwUZL2+kfeQQ", + "tg3V3gMB/YiLuuAPdlL2Bs/vvtMjziYJjRTqFQRs45GphJCfBPDCHfYQF+j3nCuMinD+R3Skrcw6Lsit", + "W15FW3/Q+JM53gkJmcHPiEgxM8kR5p01h37pOBuXRHmcV95qjvBPjjv2pnIgPOaiAkGuekT9a6suDC5f", + "R3sBBAbbp5le/ICEv3cPJ9xOtigN+5BHDgpyolySx3ScrIttXAohQVnuJVFfC80P7vPKskUE/oSn6LEQ", + "8EtSSHjlbi1dCluZyJlRgIMS4JsyYdF+911V+HtbPvGiZMCvoZuGchbK+FVxvOgjt6ZG6VcLgFgSBOYZ", + "L18rZ3p4X8sJ27mPEwYzLjxF366pb9fUqlNuqMVNAQ6md8pb2CBuZIH489kfbmx9+GZ7aG97aGV5YOTK", + "Whf+zsd9ZCNSIx4TJGc8T2I0JsjgHbnYE4VFf/oRYRHN6JwAqB0UacsTRTMsILIkRTFW2PjQGw0TK80S", + "RXNburmei0MsF7iOYyHJCHD4Rk34k2UEImWMxEh/YqH7SjjBpXLi5uwHDexFg+XViK5mXJICz48p7zaH", + "9GZptGNotj9kby3Qq15ACKZ2vEaSBOBqV9h/OEN4yOwH3zsW4gLBJE5LzoUFYAZSg0xptmU5tU2PdCQj", + "HsLaeUsYZqonMxLRCY3stC7JwsZzBhtsVXdJD9iN8/1pkbCBdjbDeG0AzxgG5z0uniFLSVX/DYMg6CjJ", + "49LJ5SCEsBjjJAkW5pgmfIyTkVmfSxLwCb6EN+yilA6X0pvEeExMCflsoWacmb/zcc5Ubv4eC34liRh2", + "NvtDBokYdq1J3C0FRHQFhdzSjOtzJnhq+twyQ9z645IsPvWH7DBOKXMUAZ/gRHJEruE7qG8FmBmGezXQ", + "gzlNYT/4US4VT33kU0d3Zpg8V1mubEaJJKobQv0cMsXRHw7b8dPWH2WPn8BZTHCs6cR7xUwJZOumUcsR", + "1rMfwasBdzuBBRh29EVqwjymAjNlYDsLcEo09bd0o6iOABVT6yscYYYynpnKEkBUM6xJrtIGYDXgJEEK", + "jpL7VgvusJMN87HQe+m4EXfPAKXVjhFl6PRH7zAN9p6Fz5MkkSChiJL/On/9K4JbWe+Bea0M1zIpHUwL", + "DCjOwXXqeNoLHM2QcVRBMcFhh8bDTuHOjTdhrLm04TK9HvgUf9BD+8F006XxD/2+bsq4Kw/Qb3+YVg70", + "WcpSgwM67HzqIu/BlKpZPi6efQgvaBN82XmFEaANc81tAifBFJBmvBvfXJGYxYjbWyBZIIxKDuQHrowp", + "w2KxKpEwsPR2BfnERDJ6i/HHECIXh52DoYtdHHa6ww5hc/jNBjgOO5/CK2C9ls2V6+A+K5ybBRE9GQw2", + "1yNh2/UN+CxbOAa+sA7YqBUVZTf1DloY1j+Xf+DfWv8sXD+Y6c5LaCKj+Dvj+yN0QHgSu6+JBlwQNbEb", + "s4gkTuxeb+i5f+eB3qyIJMl9E+hDkWfhHiuQ+h8VOcJmlcdopfn+gSlucF+XSsVs/zD0++js5wHrubWd", + "k7kLdQ7XKQEMGqtKI/MywhKdw5h651r5fgG/9u1/ne4HmIoXCZ9eHBjVHSV8ihLKbD6AF6isxQO7lvCR", + "gaEpvrOoNK5I3IaRJP71j3/CoCib/usf/7TY7v/6xz/huG8ZeDWoMX0xI1ioMcHq4gD9QkjWwwmdEzcZ", + "qAJL5kQs0O7A2vzhEfJK3VspTQ7ZkL0hKhfMy5sw9dqkbdC6CvR8KMuJtDA++kU6scVkTGxjwG7jzrJZ", + "yns90d0AHCLMwJuAvhUdDQCWHDWFtq0m2gmbTM2cK0bTepjmUrDeev6iyLUy1NszA7whg4ElDp07eGAn", + "jTbOz19s9hFoW4YqoGAQ6A5lM1aN6H/jSet5kuEoVYYCq2x4U4QzPKYJdSbHhmon5gimOJpRRsr44gJr", + "3DVx4Eaqeczh2QmygZBdeHXIXp9vgYlVkUjlgnQtJxAWYbQsh8Ztngv0APyLKogO69l3h2xCMOQJnRwb", + "JuCBcBf5gEXDDIA8IMaVqkrlte6QGSRZi1ysD17KY5LAR9D/FCtyhRddVNS6ddVREqy0Qiy7+uUhM1iv", + "dg16AFWCvGH2gZ+ZIfVcJK/N2RJkkmjVGCLwTdlv6HtjwgWyEc5elX/XnUmyNMPSi5bi6PW5nt8UNEFu", + "7IHQ0utztxubXSQ5ihIK1BBhNmRTCARy4L2cVXa1SCibYRH3Iq4vAR/M6ZLxq4TE0yYee+QT2R1KMpV+", + "Asfp5zq5PjbhYrY8AX2IDUDdas/dsX2nnevOtvhn8t3ZQpA3cN4ZCy4x/Mas7jdHXgtHXnjdnFMv5Fk7", + "dgiMdxfxa7p4oIBfR3vLa26eeEv2EBY9tOGgbcArwgU6OzpBOI4FkXLz39vep2dqqLSU//T9qFnxQ4Se", + "2LFwYUH/rL2lSiCPhR28saNG2M2rXl/Xv9+2KsV3Gm+6og5PeeXd/e1R6/Qm10gp9Ja09u0mWRtsS2XE", + "ocxgSS09EI0SUogvxTn1qWidVdmE8RZXzkpxybLnk2N3IO/Pvmy7zln9brgHpnhcY4gPyAirqdZ+1ezH", + "RM3vil10aNMrzM9fF2kO7k8Kum9TdIjMH5O6GNeWTXNBA3TSeIG+JMrAm9ylnm57CEz8nAh3qs1AF2bW", + "xbTMp8jgtMCEwBKzWvc9Ma+0U31Ne38mzReW5yYSi13ybyJKC2W3XKtVCu6JLQF9d/ot9HAj9fbLha1Y", + "AgssMlhRx87tBJbVDSwXLNr8FrnyxSnaxDWWSqxw8yZxYck2aEqFnnVfct0h8+uNa5nO6rWUoUlCpzPr", + "BIjpBGL1lF+/G0a5cw+jLOpkC6yIDVF8jHm/Z3qRrRd4ToRCr49OzPr7V+rWHxC0ul5Vcsxr5e367s2r", + "HmERjwvnSbNMap98YYXJ0H8ll/f+T90jzGelTjxoEhg/Y/9NMDky8e99yv/Xzk8JHQssFv9r5yecZJSR", + "/7V7mGBFpNq8M2IZ3NdNd98KzCMmPq2/0OqiAWtiU4CMXSPwF2+1lPnd+38qsd9M+kaCf7Gu32T/NrK/", + "v1wrxX+7FXeqAJg+HsjDVRBbaLXh0TdIm3swmlqK9CBtKl6kEtRmxqWCR48vv9kGldOC4vxro6X1vzyQ", + "K68PR7onx11YSKgoDRUtbPrgPfkC3DjuXbi1/d6/I+AwHdNpznPpZyamWEUzIm3WbkKqDPixid3l9dwo", + "eH/FVDq4z6vj3uXqb3R/RxJ/fUMN8zYOvXUyv3urrcxv39cyv0E0tZnNtuxG15Vk2mwItHaYpm3JuAL9", + "uhwAHhpXSBdB77SiUqoLCDSIgyH731r/+E0RnH74waVQ5oPBzhP4nbD5hx9cFiU7daRCmBLUVtA7/PUY", + "vKhTCJSFIntlwnZ9HKZmN5CeKyvwb6cglY7k9hqSo8JvGlIrDclbrtUakt2Lu1WRqqVJ7l1HcvQWWnCL", + "Kf7n1JL+5O6RigYn88mERpQwKPACielyKR7QaHLfPCO3TEhm1h/pBRNVJJHWamTBtdZI6GVN6XsPJDsp", + "i2jdt/boylc/zsQqntl6sFZfK6WFZoXta6OHwf3eXvevqD1mEjMa0fLSZVroDlQGMjWh0txkoRVflqBi", + "ffT27SuXcaYlfuHqTCnuiku5uptD5heX6qMXZdUu84JrQUvkJLYZqpCHZ8s1xQTHCWUEQnSJDCWHVUvC", + "Peix+PJCZbjeXSuh8p6Ppa1g+nBC5YOxgnsR304qhaF5qeP7pfCK0+JEODg1j4pfWQYUYDwh8WkL54r3", + "bA7r1owbYLMwtuNZgiOAdtSvGdQxCxtgYAb9pgALQPAkIcKgyWW5cjUQh6wYHGVejXdb4+ZCNz/KmaLJ", + "RddEyAAkiESYLSyk0pBVOsNKkTRTkNoLaeswQkEyM+Ja8Uc9aMpzCW9BFq7fJcLJFV7IIbPJwOZzKJQr", + "SGSAF5Okj37mgMOA8BRT5jFeU4HwOzlkFzROyMjCKFwgKpGccaEIIzFK+ZzIar8Ei4QSAZM4wnrlJErx", + "AvDMDLSjWR+eEYMZVgFr4PrfmMUUatnpnospHwwZRjuDAUoJZtKmXks8gQvHtoFgEJUBfY8w2hs8t1/V", + "9g0wd93yb+jTJASZ8wiPkwUimooB/EFtwgamtrakqdGrt29ChTT7VZgMbdGwysZS6Uolxl2UszK5HMzn", + "OStywfV2qVwwmKd1rBEqimvQ4mmMSYT1ejJe7QeQDHkU5SJ0Qeqt9oqc/jsKjt70zmGpwqnbCWjhEYlh", + "zxlXMzjTHI7S5vcNVFUS1Z/jogkeEi4QRh5dl0YCEuXAGjcA+e+irNjHXAXei83v3dnRx9cyAnf8Dfbe", + "Y7mfgIj4ZFI5gOuvJnOAV6VMLJPwn/WcHrlSrT6LiymeMi4VjRwzrFd2/6YQtlYIV69skJonXFz6slWV", + "fn/i4rKtBmbxROnjUsT8GX6Ftn09PMBufngTPxiYjbKiiebelbQ6fRWnFIQuqqSLHeYo4WyqT1Fp6L53", + "S7yv1W0YHDZ9mQrjPy5Qc7QSMrI/mmqvejK2liZY7SPb6kPzIt37Pfh3fuUK0TRLSEqgGmzPEJve7BJh", + "CSrnU+nhDN2MV+pT5acDG11QGpd+14lDQFduwzZAel/eriBTTfh0PY5f0bkDrQsA+Q2ZqexP0IXx5lyg", + "ggdrgdag5qOrGY1mAOoHeqtu32D+4Sy7KPCMNw/QSzjIPqwzdL5hsPI1rUmeEIPVN0/Ti4PleqfvT0/h", + "I4PnZyqbXhwgV+O0uD+kfssH6dOzSLBU6FcLPbhRKOOwoxcKa32zmN+mhe8r8aaHLATlx8iVbZBO0IWH", + "6nfRADnl+O0rPn0wYazbXCXAzEVxZFVHoE3C4k5T3ARNwoB+24NBCLy6JbigGcYdYwsuDeYVnxYVCiqk", + "jLOsLfnaYQIVz9N0BQ2jDQ+pTKqY5+ovUsVECPjYUncTcaMNHNnqVPhSE6rFpXMHexPILxgdZCDDg0ul", + "mWqn2yEsTzsHv9l/zdO00+3Y8XhQ4zcQ7teANNYbXI5i0TvjITF+E8tvgrFYZfYeyGLt5rDqdLNE/sa8", + "8Kf3Fjqb3QOSIcgHNSPu1ySCeuOtGnwYL8AiYWTP72NkgCiJooRLUnHwPB48KmvoqsmMzYYit8Y9Pbw4", + "dwV82gSFnNtPz92XX4HuvS5WxI0Zuenee9DI8ggec26tXJrNhIs6iNG6aJKvnpC+3JYsTbUNhXyjzZtb", + "GVsRptYTllmE/SA2Bd1wrniKFY2gmFA041x6ZF8gDpuyX9Z4XFAmmFaMlmuD8i80qV5YM/SFVSMOrMkM", + "Yf+R7aMPn9tQ/vAX7lH5xU+eVaDg+F0n+gPgPlQ7F5RMUIZzSbRUl6cERYtIc0VTPYrgaIYinKlcECiM", + "R1BKGU3z1IeS1js2xwB7cbGdXnTROFcowWIKWpl56IJtIp6mhMUE7HNDNiN4TrVKKVCCFWHRoicJFNSd", + "E3TFxWXCcQwmhizG4OmBgnyCaAoEXO6UKBxjhUHQudAnfmTygi6KGrtGrWfkuqSGeMhEzr43RQJ0sxdu", + "oBeIAAo2lbOiFmOEY8KiIDr0+dfNxr68LfqcqPpEHygy6Fa89CFDhXybqxvO1xFF9MjCm7mw29iGza8Q", + "emWzCltNqHBk9O95pM1c3RwfyMFULPGqU/x1eJYKovtqvEsP7z7iAsW56c47lUDmf1afUMFQ/GArSNY0", + "23hbx1BRdK5Y5hvxvK0/3J8nt7DlfSWcsNuo2DeVNyon/TWwXLuqt+K5D2TEtLYk3yb3cCzYRXQ9mPjE", + "hcflHoux1TJsczQLvu1zJyUwaF+cfWPbdbZtAx5uy7adbXbJpe8xcsp6ECMa5uDWjNvIqq3p4N80G6U2", + "O49lPjiLLD0X98YWTwpGaFhjhhcJx/GfIUh4hf8o4kIYRAnAqHhMiKae1dBPDwDbXFk3reuyNd+fnm42", + "cQmhVvIIoR4xh/BScvRnabxswH09J0LQ2AJ/oqPTYxuuSyUSOeuj1ylVSHF0SUhWZrRAVmFfz89hayzX", + "aK+AaHQ7hCmxyDhlau0oylfvZjCfblXZ/Z75pIXI/uYOb+0OB8v+42NnwGUgZ8NMYLVmqrBaW7qTsgkX", + "qZHL8JjnunXNg/Qy6f00JTAnNCFyIRVJTVTiJE/guEG5BVtS135ndrkLMbn65Jh0uYyIlEpJOZNDZnNF", + "MiJ03/pz3b4XYBV0CChc8NczwyS/juA9PRgTr4ZV06oBChKU6uwcdLZwlm3FWOGGADE7vM8Y0k8QjYfk", + "Ih3zhEYooexSoo2EXhr1BM0lSvQfmyvD+Ubw3ZcuGHz7k6VX+oRNeLAcm6HZgpj/VFldlq05x+SjY2sv", + "iX9YHP+BjQ6ztfUliQXBSQ9K/DosHJQrmtCPhtXpRqhUNDIpR7hYu/enBVPtD9kpUUK/gyG1LUkMogFo", + "l1uZ4NHWMB8MdqOMAqDaLoHBAcNrfpxCj0dn70waKkm5WHSHTP8DGn57eGa8uxNsrQneQG0tYnSy9XpN", + "gPM5LNO/cYSgmeBK9ILghn9zCd4cY6TxDMmGI8qzVaoSz/70IaxWgvtmV3icdgUAeSpmszEVOAKhWM5y", + "FfMrFrYhzHmSp/of5o+TdVBhCkez9/DqVyPtmuGs7cZN8FEcSjunmJhykQ/i9DAL9lhjVvXCuSmAEFOJ", + "BgzeAofqz0jdX95876/jV+jutCvqSrF+NWfrvm8+OwaHsOGvx2M55obS3EwUX219usK02fr0Y8KjS2mh", + "WHyzodbbALJc/1hCTFsXIYgJkBmKLISRAcoisjtkNQOkQfyRCCNFREoZTrZgzqYRAMt2Viw85xQStCPI", + "U+lJGgNmUgKI2AB/p2cDhirXgOfRlbZYlf+O74xUHI1JxFPiAMQ3Q6rb3zBVP3FRRQP/WvjiW2/9ARIQ", + "U7C3rwFAb+7xswDRT/E1hErHuXUouxFtvOTlj8YU1EWwN8PO7kAOO1007Oykw47egSMMJlSs0D5KKcsV", + "kX10bOxbkIL7ZIAkiTiLpcMxdxa83YFsSsg1ZNmQ3fkEvrtPscdSFSzlG9tJiD3o95D+HpJ20IZ/4OyZ", + "jLtw6GLEc2XM/fZc2bdiosA8snnvvlrvjHzT7dtw8r/Z41vhUbDLml16W284e5bLGWk2ub0ytYFyNQZ8", + "bFevU87Q3/lYdhEjV8YaLqTqL/E9/fWZ6eA+sPt1VzfB7bdz/wba3wK0v1yrMFijCbDUV7KjDoPYSK4z", + "LhSgONpce0NDoEkAcgSPcIJeH50MWaRZkYEWFCTlwJ0sxLi5hQ//do5eHL3pomOoHYl+zsebffSaJQtX", + "wdv4aIbMSGKGeUWYobGhWhKHrmczdqCeuwwW1x08UDFmczICnhW3Vy5IvNuZERyDRPJH5xU3nQVQh9+8", + "0gcIgH/Nl8W2d1YKH503RIlF73CiiFhu9tTmSbECM8Ne0g6CzgpuBvhSdygd8lrZp5ENDDTG7k4ngJTx", + "6VsdhbuvOXo/XjITJ2Iq2I1zQBplkGSA48XjimWSM1QwxxAL9K/rohJBU5aw5WUrFQzosiny+ysyua/k", + "XRVs+X/X0wUzfbSOpqyyT5qIiwomaz29Ljl4ZuCQraMqwhmOqFp0EU4Se0fZm6CISOkV4u9YEHwZ8yvW", + "H7I3Re0Um9CLjs7edZ2jFsVUXpoWrC+2j17PiZD5uBgcgoNmvMaw5iQeMsVRhJMoT7S4QSYTEkEuLpRE", + "kQ2+3GIonTs8O2UnwfotXlR7/ujKxoVpAnavJIs6xW2Zrd4SJEowTZvBx62gBgGHEGow1o1yhiibJDak", + "KhJcSmSb6pGETuk4sQFCso/ezgiSOCVDliWYMSJQLk1UvB56LxNEytwkeOsGAKTXUFQXlcCCmeDKhiYk", + "nAtpogk0hb8/RVKRbAWZvTEtn8Kc70i2NY3bnh7ISF0bQ7MpxL6C9IYYSjELrukoT1wA472GopsBPbSU", + "+FgO/ltBp1Mi9KnAhsmacDxzrN1ymkNfyVhuLCF5XrzVroRk0aqXlehl7K0EhhuVWNtx52ZRf4HOL2kj", + "dqB9dLMs4l/0Ry37rmarhgdhH33mLP8slfnPvSTBtgasksIfmznJG3nlqFYSbdfDarXOrL3LTNfW+FkP", + "Bpv1mNGycCV9tknh/foIYXC/KA/3XWTtcdNWBe2qops2pPyvR9P/KijwbmD0Hxjl5BYw+l9V3j3gnD8c", + "/knwoD5UHn3F9+zq1/7pkfDvKn3ewOEDHFtT+rzhejZ4daWi9N6+005Nsi3+mSR4G+94A/ndLfs3rb+F", + "yuAt1joXtCZ4kmZq4QLarK+yDDqT9CPpNziCi7jVu3MF3yKk88uRh6PTxoDOP2e5+QeJGbWlA6lEJ8eB", + "Ou6PDGPQP3OVi2VL3zo9LKIZnZNmo3v1BNslygTpZTwD50psFsyuh7vLFBb96Udkm7eYq/ZfUHsSoPpJ", + "jGIqSKSShakDqjmC6eM7iQTXmgA852LRHCVijshPgqeHdjZr7kN7pqwxrIwzTBe9GCvcmztus8KE9hnR", + "nS6eUjM8RBl6+SPaINdKmAoXaKI1H0QnxZKaAv4SaHLTH/D2oMGyST+S0XTcZpQrapW8trVgUJRLxVO3", + "9yfHaANqn00J03uhRf0JSLKZ4HMak7gyxs6cJ2ZVtxsW9KZ2Vy1UFIXrnHJhBvcgMkybC2n6kWZVtlCE", + "xIwpwzC4tVVBqmfKJPHr/jBlLgDH7pEbxbcrzGp+G07Z0ZQIdTjtIirODcTz5rdr7jFfc34ylLvTKred", + "C89Zbbxulx/VMm3pLgo/FLlz92u2fv/1pPRQ+SizeazpfF4opE1m86+LBAf3dz/ct7n8/SNOAX1JnPLt", + "mcqhAd1iiGBeQUx3TOYk4VkK9dDh3U63k4ukc9CZKZUdbG1B7PeMS3Ww9/zpbufTh0//fwAAAP//5962", + "rgXyAQA=", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/lib/paths/paths.go b/lib/paths/paths.go index add23ff52..aff107dca 100644 --- a/lib/paths/paths.go +++ b/lib/paths/paths.go @@ -314,6 +314,11 @@ func (p *Paths) DeviceMetadata(id string) string { return filepath.Join(p.DeviceDir(id), "metadata.json") } +// VFHealthState returns the path to the persisted vGPU VF health file. +func (p *Paths) VFHealthState() string { + return filepath.Join(p.dataDir, "gpu", "vf-health.json") +} + // Volume path methods // VolumesDir returns the root volumes directory. diff --git a/lib/providers/vgpu_sentinel.go b/lib/providers/vgpu_sentinel.go new file mode 100644 index 000000000..0931f1fe6 --- /dev/null +++ b/lib/providers/vgpu_sentinel.go @@ -0,0 +1,13 @@ +package providers + +import ( + "log/slog" + + "github.com/kernel/hypeman/lib/instances" + "go.opentelemetry.io/otel" +) + +func ProvideVGPUSentinelController(instanceManager instances.Manager, log *slog.Logger) (*instances.VGPUSentinelController, error) { + meter := otel.GetMeterProvider().Meter("hypeman") + return instances.NewVGPUSentinelController(instanceManager, meter, log) +} diff --git a/lib/resources/gpu.go b/lib/resources/gpu.go index 054e3744e..f4878c8d2 100644 --- a/lib/resources/gpu.go +++ b/lib/resources/gpu.go @@ -10,11 +10,14 @@ import ( // GPUResourceStatus represents the GPU resource status for the API response. // Returns nil if no GPU is available on the host. type GPUResourceStatus struct { - Mode string `json:"mode"` // "vgpu" or "passthrough" - TotalSlots int `json:"total_slots"` // VFs for vGPU, physical GPUs for passthrough - UsedSlots int `json:"used_slots"` // Slots currently in use - Profiles []devices.GPUProfile `json:"profiles,omitempty"` // vGPU mode only - Devices []devices.PassthroughDevice `json:"devices,omitempty"` // passthrough mode only + Mode string `json:"mode"` // "vgpu" or "passthrough" + TotalSlots int `json:"total_slots"` // VFs for vGPU, physical GPUs for passthrough + UsedSlots int `json:"used_slots"` // Slots currently in use, including assigned quarantined VFs + AllocatableSlots int `json:"allocatable_slots"` // Healthy free slots used by admission control + QuarantinedSlots int `json:"quarantined_slots"` // Quarantined VFs; may overlap UsedSlots + allocatableSlotsErr error + Profiles []devices.GPUProfile `json:"profiles,omitempty"` // vGPU mode only + Devices []devices.PassthroughDevice `json:"devices,omitempty"` // passthrough mode only } // GetGPUStatus returns the current GPU resource status. @@ -51,13 +54,21 @@ func getVGPUStatus(ctx context.Context, framework devices.VGPUFramework, vfs []d logger.FromContext(ctx).WarnContext(ctx, "failed to list vGPU profiles; reporting none", "framework", framework, "error", err) profiles = nil } - - return &GPUResourceStatus{ - Mode: string(devices.GPUModeVGPU), - TotalSlots: len(vfs), - UsedSlots: usedSlots, - Profiles: profiles, + allocatableSlots, quarantinedSlots, err := devices.VGPUAvailability(framework, vfs) + status := &GPUResourceStatus{ + Mode: string(devices.GPUModeVGPU), + TotalSlots: len(vfs), + UsedSlots: usedSlots, + AllocatableSlots: allocatableSlots, + QuarantinedSlots: quarantinedSlots, + Profiles: profiles, + } + if err != nil { + logger.FromContext(ctx).WarnContext(ctx, "failed to count allocatable vGPU slots; reporting none", "framework", framework, "error", err) + status.AllocatableSlots = 0 + status.allocatableSlotsErr = err } + return status } // getPassthroughStatus returns GPU status for whole-GPU passthrough mode. @@ -92,9 +103,10 @@ func getPassthroughStatus() *GPUResourceStatus { } return &GPUResourceStatus{ - Mode: string(devices.GPUModePassthrough), - TotalSlots: len(passthroughDevices), - UsedSlots: usedSlots, - Devices: passthroughDevices, + Mode: string(devices.GPUModePassthrough), + TotalSlots: len(passthroughDevices), + UsedSlots: usedSlots, + AllocatableSlots: len(passthroughDevices) - usedSlots, + Devices: passthroughDevices, } } diff --git a/lib/resources/gpu_test.go b/lib/resources/gpu_test.go new file mode 100644 index 000000000..4eb16d7c6 --- /dev/null +++ b/lib/resources/gpu_test.go @@ -0,0 +1,85 @@ +package resources + +import ( + "context" + "errors" + "os" + "path/filepath" + "testing" + + "github.com/kernel/hypeman/cmd/api/config" + "github.com/kernel/hypeman/lib/devices" + "github.com/kernel/hypeman/lib/paths" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func initVFHealthForTest(t *testing.T, state []byte) { + t.Helper() + dataDir := t.TempDir() + if state != nil { + path := paths.New(dataDir).VFHealthState() + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o755)) + require.NoError(t, os.WriteFile(path, state, 0o644)) + } + devices.NewManager(paths.New(dataDir)) + resetDir := t.TempDir() + t.Cleanup(func() { devices.NewManager(paths.New(resetDir)) }) +} + +func TestGetVGPUStatusFailsClosedWhenVFHealthIsUnavailable(t *testing.T) { + initVFHealthForTest(t, []byte("not json")) + + status := getVGPUStatus(context.Background(), devices.VGPUFrameworkVendorVFIO, []devices.VirtualFunction{{PCIAddress: "0000:82:00.4"}}) + assert.Zero(t, status.AllocatableSlots) + assert.Zero(t, status.QuarantinedSlots) + require.ErrorContains(t, status.allocatableSlotsErr, "VF health state unavailable") +} + +func TestGetVGPUStatusReportsQuarantinedSlots(t *testing.T) { + initVFHealthForTest(t, nil) + for _, instance := range []string{"instance-1", "instance-2"} { + _, err := devices.ReportVFInitFailure(devices.VFInitFailureReport{VFAddress: "0000:82:00.4", InstanceID: instance}) + require.NoError(t, err) + } + + status := getVGPUStatus(context.Background(), devices.VGPUFrameworkVendorVFIO, []devices.VirtualFunction{ + {PCIAddress: "0000:82:00.4"}, + {PCIAddress: "0000:82:00.5", Allocated: true}, + {PCIAddress: "0000:82:00.6"}, + }) + assert.Equal(t, 3, status.TotalSlots) + assert.Equal(t, 1, status.UsedSlots) + assert.Equal(t, 1, status.AllocatableSlots) + assert.Equal(t, 1, status.QuarantinedSlots) +} + +func TestReserveAllocationUsesAllocatableGPUSlots(t *testing.T) { + status := &GPUResourceStatus{ + Mode: string(devices.GPUModeVGPU), + TotalSlots: 4, + UsedSlots: 1, + AllocatableSlots: 0, + } + setGPUStatusProvider(func(context.Context) *GPUResourceStatus { return status }) + t.Cleanup(func() { setGPUStatusProvider(nil) }) + + mgr := NewManager(&config.Config{}, paths.New(t.TempDir())) + ctx := context.Background() + + err := mgr.ValidateAllocation(ctx, 0, 0, 0, 0, 0, 0, true) + require.ErrorContains(t, err, "no allocatable vgpu slots") + + status.allocatableSlotsErr = errors.New("VF health state unavailable: read failed") + err = mgr.ValidateAllocation(ctx, 0, 0, 0, 0, 0, 0, true) + require.ErrorContains(t, err, "vGPU placement is disabled: VF health state unavailable") + status.allocatableSlotsErr = nil + + status.AllocatableSlots = 1 + require.NoError(t, mgr.ReserveAllocation(ctx, "pending-a", 0, 0, 0, 0, 0, 0, true)) + err = mgr.ReserveAllocation(ctx, "pending-b", 0, 0, 0, 0, 0, 0, true) + require.ErrorContains(t, err, "no allocatable vgpu slots") + + mgr.FinishAllocation("pending-a") + require.NoError(t, mgr.ReserveAllocation(ctx, "pending-b", 0, 0, 0, 0, 0, 0, true)) +} diff --git a/lib/resources/resource.go b/lib/resources/resource.go index 86f644bda..8c5d44a51 100644 --- a/lib/resources/resource.go +++ b/lib/resources/resource.go @@ -695,11 +695,14 @@ func (m *Manager) validateAllocationLocked(ctx context.Context, excludeID string if gpuStatus == nil { return fmt.Errorf("insufficient GPU: no GPU available on this host") } - availableSlots := gpuStatus.TotalSlots - gpuStatus.UsedSlots - pending.GPUSlots + availableSlots := gpuStatus.AllocatableSlots - pending.GPUSlots if availableSlots < req.GPUSlots { + if gpuStatus.allocatableSlotsErr != nil { + return fmt.Errorf("insufficient GPU: vGPU placement is disabled: %w", gpuStatus.allocatableSlotsErr) + } if availableSlots <= 0 { - return fmt.Errorf("insufficient GPU: all %d %s slots are in use", - gpuStatus.TotalSlots, gpuStatus.Mode) + return fmt.Errorf("insufficient GPU: no allocatable %s slots available (%d total, %d in use)", + gpuStatus.Mode, gpuStatus.TotalSlots, gpuStatus.UsedSlots) } return fmt.Errorf("insufficient GPU: requested %d %s slot(s), but only %d available", req.GPUSlots, gpuStatus.Mode, availableSlots) diff --git a/lib/system/guest_agent/gpu_watch.go b/lib/system/guest_agent/gpu_watch.go new file mode 100644 index 000000000..0decdea6c --- /dev/null +++ b/lib/system/guest_agent/gpu_watch.go @@ -0,0 +1,210 @@ +package main + +import ( + "bufio" + "context" + "errors" + "io" + "log" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "sync" + "syscall" + "time" +) + +const ( + gpuInitFailedSentinelPrefix = "HYPEMAN-GPU-INIT-FAILED" + gpuInitOKSentinelPrefix = "HYPEMAN-GPU-INIT-OK" + + kmsgPath = "/dev/kmsg" + nvidiaPCIVendorID = "0x10de" + + gpuProbeInterval = 15 * time.Second + gpuProbeTimeout = 10 * time.Minute + // nvidia-smi can hang indefinitely on a wedged VF; without a per-attempt + // bound the overall probe deadline is never reached. + gpuProbeAttemptTimeout = 30 * time.Second + + gpuReportThrottle = 30 * time.Second + + gpuReportRepeats = 3 + + kmsgReopenDelay = 5 * time.Second + + // /dev/kmsg returns EINVAL without consuming records larger than this buffer. + kmsgRecordBufferBytes = 8192 + + kmsgOpenRetryDelay = time.Minute +) + +func hasNVIDIADevice() bool { + vendors, _ := filepath.Glob("/sys/bus/pci/devices/*/vendor") + for _, path := range vendors { + data, err := os.ReadFile(path) + if err != nil { + continue + } + if strings.TrimSpace(string(data)) == nvidiaPCIVendorID { + return true + } + } + return false +} + +type gpuInitReporter struct { + mu sync.Mutex + succeeded bool + lastFailure time.Time +} + +func (r *gpuInitReporter) reportFailure(msg string) { + r.mu.Lock() + defer r.mu.Unlock() + if r.succeeded || time.Since(r.lastFailure) < gpuReportThrottle { + return + } + r.lastFailure = time.Now() + emitGPUInitFailureReport(msg) +} + +func (r *gpuInitReporter) reportSuccess() { + r.mu.Lock() + defer r.mu.Unlock() + if r.succeeded { + return + } + r.succeeded = true + emitGPUInitOKReport() +} + +func watchGPUInitFailure(reporter *gpuInitReporter) { + firstOpen := true + for { + f, err := os.Open(kmsgPath) + if err != nil { + log.Printf("[guest-agent] cannot open %s for GPU init watch (retrying): %v", kmsgPath, err) + time.Sleep(kmsgOpenRetryDelay) + continue + } + // A reopened fd restarts at the oldest record; skip history already + // scanned so a stale failure line is not reported again. + if !firstOpen { + if _, err := f.Seek(0, io.SeekEnd); err != nil { + log.Printf("[guest-agent] cannot seek %s to end (may re-report old records): %v", kmsgPath, err) + } + } + firstOpen = false + err = scanKmsg(f, reporter.reportFailure) + _ = f.Close() + if err != nil { + log.Printf("[guest-agent] GPU init watch read %s failed (reopening): %v", kmsgPath, err) + } + time.Sleep(kmsgReopenDelay) + } +} + +func emitGPUInitFailureReport(msg string) { + ts := time.Now().UTC().Format(time.RFC3339Nano) + for range gpuReportRepeats { + log.Printf("[guest-agent] %s ts=%s nvrm=%q", gpuInitFailedSentinelPrefix, ts, msg) + } +} + +// probeGPUInit reports successful guest driver init. Opening the device via +// nvidia-smi runs RmInitAdapter, so on a wedged VF the probe itself produces +// the failure line in kmsg without waiting for the workload to touch the GPU. +func probeGPUInit(reporter *gpuInitReporter) { + nvidiaSMI, err := exec.LookPath("nvidia-smi") + if err != nil { + return + } + deadline := time.Now().Add(gpuProbeTimeout) + for { + err := runGPUProbeAttempt(nvidiaSMI, gpuProbeAttemptTimeout) + if err == nil { + reporter.reportSuccess() + return + } + if errors.Is(err, context.DeadlineExceeded) { + log.Printf("[guest-agent] GPU init probe attempt timed out after %s", gpuProbeAttemptTimeout) + return + } + if time.Now().After(deadline) { + log.Printf("[guest-agent] GPU init probe gave up after %s", gpuProbeTimeout) + return + } + time.Sleep(gpuProbeInterval) + } +} + +func runGPUProbeAttempt(nvidiaSMI string, timeout time.Duration) error { + cmd := exec.Command(nvidiaSMI, "-L") + if err := cmd.Start(); err != nil { + return err + } + done := make(chan error, 1) + go func() { done <- cmd.Wait() }() + + timer := time.NewTimer(timeout) + defer timer.Stop() + select { + case err := <-done: + return err + case <-timer.C: + _ = cmd.Process.Kill() + return context.DeadlineExceeded + } +} + +func emitGPUInitOKReport() { + ts := time.Now().UTC().Format(time.RFC3339Nano) + for range gpuReportRepeats { + log.Printf("[guest-agent] %s ts=%s", gpuInitOKSentinelPrefix, ts) + } +} + +func scanKmsg(r io.Reader, report func(msg string)) error { + reader := bufio.NewReaderSize(r, kmsgRecordBufferBytes) + for { + record, err := reader.ReadString('\n') + if msg, ok := gpuInitFailureMessage(record); ok { + report(msg) + } + if err != nil { + if errors.Is(err, io.EOF) { + return nil + } + // EPIPE means records were overwritten while reading; the fd + // continues at the next available record. + if errors.Is(err, syscall.EPIPE) { + continue + } + return err + } + } +} + +// Only kernel-facility records match; userspace /dev/kmsg writes use LOG_USER. +func gpuInitFailureMessage(record string) (string, bool) { + prefix, msg, found := strings.Cut(record, ";") + if !found { + return "", false + } + priority, _, found := strings.Cut(prefix, ",") + if !found { + return "", false + } + value, err := strconv.ParseUint(priority, 10, 32) + if err != nil || value>>3 != 0 { + return "", false + } + msg = strings.TrimSpace(msg) + if !strings.HasPrefix(msg, "NVRM:") || !strings.Contains(msg, "RmInitAdapter failed!") { + return "", false + } + return msg, true +} diff --git a/lib/system/guest_agent/gpu_watch_test.go b/lib/system/guest_agent/gpu_watch_test.go new file mode 100644 index 000000000..c253c62cf --- /dev/null +++ b/lib/system/guest_agent/gpu_watch_test.go @@ -0,0 +1,194 @@ +package main + +import ( + "bytes" + "context" + "io" + "log" + "os" + "path/filepath" + "strings" + "syscall" + "testing" + "time" + + "github.com/kernel/hypeman/lib/instances" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestGPUInitFailureMessage(t *testing.T) { + msg, ok := gpuInitFailureMessage("3,1042,8462102,-;NVRM: GPU 0000:00:03.0: RmInitAdapter failed! (0x22:0x65:884)\n") + assert.True(t, ok) + assert.Equal(t, "NVRM: GPU 0000:00:03.0: RmInitAdapter failed! (0x22:0x65:884)", msg) + + _, ok = gpuInitFailureMessage("3,1042,8462102,-;NVRM: GPU 0000:00:03.0: RmInitAdapter failed! (0x26:0xffff:1482)\n") + assert.True(t, ok) + + _, ok = gpuInitFailureMessage("4,1044,8462120,-;NVRM: GPU 0000:00:03.0: RmInitAdapter failed! (0x22:0x65:884)\n") + assert.True(t, ok) + + for _, record := range []string{ + "6,1041,8462100,-;NVRM: loading NVIDIA UNIX Open Kernel Module for x86_64\n", + "6,1043,8462110,-;nvidia-gridd: RmInitAdapter failed mentioned in userspace\n", + "no separator RmInitAdapter failed!\n", + " continuation line of a multi-line record\n", + "12,307,4250363151,-;NVRM: GPU 0000:00:03.0: RmInitAdapter failed! (0x22:0x65:884)\n", // plain write + "8,308,4250380620,-;NVRM: GPU 0000:00:03.0: RmInitAdapter failed! (0x22:0x65:884)\n", // "<0>" prefix + "9,310,5898419120,-;NVRM: GPU 0000:00:03.0: RmInitAdapter failed! (0x22:0x65:884)\n", // "<1>" prefix + "24,309,5898400480,-;NVRM: GPU 0000:00:03.0: RmInitAdapter failed! (0x22:0x65:884)\n", // "<24>" prefix (facility 3) + "x,1,100,-;NVRM: GPU 0000:00:03.0: RmInitAdapter failed! (0x22:0x65:884)\n", // malformed priority + } { + _, ok := gpuInitFailureMessage(record) + assert.False(t, ok, "record %q must not match", record) + } +} + +func captureAgentLog(t *testing.T) *bytes.Buffer { + t.Helper() + var buf bytes.Buffer + prevOutput := log.Writer() + prevFlags := log.Flags() + log.SetOutput(&buf) + log.SetFlags(log.LstdFlags) + t.Cleanup(func() { + log.SetOutput(prevOutput) + log.SetFlags(prevFlags) + }) + return &buf +} + +func TestEmitGPUInitReportsMatchHostSentinel(t *testing.T) { + tests := []struct { + name string + emit func() + event instances.VGPUSentinelEvent + }{ + { + name: "failure", + emit: func() { + emitGPUInitFailureReport("NVRM: GPU 0000:00:03.0: RmInitAdapter failed! (0x22:0x65:884)") + }, + event: instances.VGPUSentinelEventInitFailed, + }, + {name: "success", emit: emitGPUInitOKReport, event: instances.VGPUSentinelEventInitOK}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + buf := captureAgentLog(t) + tt.emit() + lines := strings.Split(strings.TrimSpace(buf.String()), "\n") + require.Len(t, lines, gpuReportRepeats) + for _, line := range lines { + _, event := instances.MatchVGPUSentinelLine([]byte(line)) + require.Equal(t, tt.event, event, "host matcher rejected %q", line) + } + }) + } +} + +func TestProbeGPUInitEmitsOKOnceDriverResponds(t *testing.T) { + buf := captureAgentLog(t) + + binDir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(binDir, "nvidia-smi"), []byte("#!/bin/sh\nexit 0\n"), 0o755)) + t.Setenv("PATH", binDir) + + probeGPUInit(&gpuInitReporter{}) + + assert.Contains(t, buf.String(), gpuInitOKSentinelPrefix) +} + +func TestProbeGPUInitSkipsWithoutNvidiaSMI(t *testing.T) { + buf := captureAgentLog(t) + t.Setenv("PATH", t.TempDir()) + + probeGPUInit(&gpuInitReporter{}) + + assert.Empty(t, buf.String()) +} + +func TestGPUInitReporterMakesSuccessTerminal(t *testing.T) { + buf := captureAgentLog(t) + reporter := &gpuInitReporter{} + reporter.reportFailure("NVRM: GPU 0000:00:03.0: RmInitAdapter failed!") + reporter.reportSuccess() + reporter.reportFailure("NVRM: GPU 0000:00:03.0: RmInitAdapter failed!") + + output := buf.String() + assert.Equal(t, gpuReportRepeats, strings.Count(output, gpuInitFailedSentinelPrefix)) + assert.Equal(t, gpuReportRepeats, strings.Count(output, gpuInitOKSentinelPrefix)) + assert.Less(t, strings.Index(output, gpuInitFailedSentinelPrefix), strings.Index(output, gpuInitOKSentinelPrefix)) +} + +func TestRunGPUProbeAttemptReturnsOnTimeout(t *testing.T) { + path := filepath.Join(t.TempDir(), "nvidia-smi") + require.NoError(t, os.WriteFile(path, []byte("#!/bin/sh\nexec sleep 10\n"), 0o755)) + + start := time.Now() + err := runGPUProbeAttempt(path, 10*time.Millisecond) + require.ErrorIs(t, err, context.DeadlineExceeded) + assert.Less(t, time.Since(start), time.Second) +} + +func TestScanKmsgReportsEachFailureRecord(t *testing.T) { + records := strings.Join([]string{ + "6,1,100,-;booting", + "3,2,200,-;NVRM: GPU 0000:00:03.0: RmInitAdapter failed! (0x22:0x65:884)", + "6,3,300,-;unrelated", + "3,4,400,-;NVRM: GPU 0000:00:03.0: RmInitAdapter failed! (0x22:0x65:884)", + }, "\n") + "\n" + + var got []string + require.NoError(t, scanKmsg(strings.NewReader(records), func(msg string) { got = append(got, msg) })) + assert.Len(t, got, 2) +} + +// kmsgOverrun marks a read that fails with EPIPE, as /dev/kmsg does when +// records are overwritten while being read. +const kmsgOverrun = "\x00overrun" + +type kmsgConn struct { + records []string + pos int +} + +func (k *kmsgConn) Read(p []byte) (int, error) { + if k.pos >= len(k.records) { + return 0, io.EOF + } + rec := k.records[k.pos] + if rec == kmsgOverrun { + k.pos++ + return 0, syscall.EPIPE + } + if len(p) < len(rec) { + return 0, syscall.EINVAL + } + k.pos++ + return copy(p, rec), nil +} + +func TestScanKmsgReadsOversizedRecords(t *testing.T) { + oversized := "6,1,100,-;" + strings.Repeat("x", 5000) + "\n" + failure := "3,2,200,-;NVRM: GPU 0000:00:03.0: RmInitAdapter failed! (0x22:0x65:884)\n" + + var got []string + require.NoError(t, scanKmsg(&kmsgConn{records: []string{oversized, failure}}, + func(msg string) { got = append(got, msg) })) + assert.Len(t, got, 1) + + huge := "6,3,300,-;" + strings.Repeat("x", kmsgRecordBufferBytes) + "\n" + err := scanKmsg(&kmsgConn{records: []string{huge}}, func(string) {}) + assert.ErrorIs(t, err, syscall.EINVAL) +} + +func TestScanKmsgContinuesAfterOverrun(t *testing.T) { + failure := "3,2,200,-;NVRM: GPU 0000:00:03.0: RmInitAdapter failed! (0x22:0x65:884)\n" + + var got []string + require.NoError(t, scanKmsg(&kmsgConn{records: []string{kmsgOverrun, failure}}, + func(msg string) { got = append(got, msg) })) + assert.Len(t, got, 1, "records after an overrun must still be scanned on the same fd") +} diff --git a/lib/system/guest_agent/main.go b/lib/system/guest_agent/main.go index 84fd2a5da..870db8182 100644 --- a/lib/system/guest_agent/main.go +++ b/lib/system/guest_agent/main.go @@ -54,6 +54,12 @@ func main() { startClockKeeper() + if hasNVIDIADevice() { + reporter := &gpuInitReporter{} + go watchGPUInitFailure(reporter) + go probeGPUInit(reporter) + } + // Create gRPC server grpcServer := grpc.NewServer() pb.RegisterGuestServiceServer(grpcServer, &guestServer{}) diff --git a/openapi.yaml b/openapi.yaml index 002c9fcce..eb31e5863 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -1800,7 +1800,7 @@ components: type: object description: GPU resource status. Null if no GPUs available. nullable: true - required: [mode, total_slots, used_slots] + required: [mode, total_slots, used_slots, allocatable_slots, quarantined_slots] properties: mode: type: string @@ -1813,8 +1813,16 @@ components: example: 64 used_slots: type: integer - description: Slots currently in use + description: Slots currently in use. Includes quarantined VFs that are still assigned, so this can overlap quarantined_slots. example: 5 + allocatable_slots: + type: integer + description: Free slots eligible for placement, matching admission control (excludes quarantined VFs; 0 while VF health state is unavailable) + example: 57 + quarantined_slots: + type: integer + description: VFs quarantined after guest driver init failures (vGPU mode only). May overlap used_slots until the affected instance releases its VF. + example: 2 profiles: type: array description: Available vGPU profiles (only in vGPU mode)