From e6034291a57046ce9fa67c3f42219ce9fdf20779 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Thu, 6 Aug 2026 19:05:18 +0000 Subject: [PATCH 01/76] Guard vGPU releases with live-instance claims A vGPU assignment goes stale when its release succeeds but the metadata save does not (or start fails between the release and its first save). The backend's owner map only covers assignments created since the last restart and the VFIO handle scan only covers VMs that have opened the device, so after a restart a stale release could still clear a VF during another live instance's pre-open boot window. Consult live instance metadata on every release: when another instance with a live hypervisor process claims the same device path, drop the stale metadata without touching the device. Tag assignments with the owning instance ID, persist the assignment before booting a started instance, and retain assignment metadata when rollback release fails in create and start so later release paths can still find the device. --- lib/instances/create.go | 41 ++++++++++++++++++++- lib/instances/delete.go | 2 +- lib/instances/lifecycle_noop_test.go | 55 ++++++++++++++++++++++++++++ lib/instances/start.go | 9 ++++- lib/instances/stop.go | 2 +- lib/instances/vgpu.go | 44 +++++++++++++++++----- lib/instances/vgpu_test.go | 50 ++++++++++++++++++++----- 7 files changed, 180 insertions(+), 23 deletions(-) diff --git a/lib/instances/create.go b/lib/instances/create.go index b4b0952da..2608b2bd2 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -277,11 +277,13 @@ func (m *manager) createInstance( var gpuFramework devices.VGPUFramework var gpuDevicePath string var gpuMdevUUID string + var stored *StoredMetadata + var retainedVGPU *StoredMetadata // Setup cleanup stack early so device attachment errors trigger cleanup cu := cleanup.Make(func() { log.DebugContext(ctx, "cleaning up instance on error", "instance_id", id) - m.deleteInstanceData(id) + m.cleanupFailedCreate(ctx, id, retainedVGPU) }) defer cu.Clean() @@ -320,6 +322,25 @@ func (m *manager) createInstance( } if err := devices.DestroyVGPU(ctx, assignment); err != nil { log.WarnContext(ctx, "failed to destroy vGPU on cleanup", "instance_id", id, "uuid", gpuDevice.MdevUUID, "error", err) + retainedVGPU = stored + if retainedVGPU == nil { + retainedVGPU = &StoredMetadata{ + Id: id, + Name: req.Name, + Image: req.Image, + ResolvedImage: resolvedImageRef, + Platform: imageInfo.Platform, + CreatedAt: time.Now(), + HypervisorType: hvType, + HypervisorVersion: hvVersion, + SocketPath: m.paths.InstanceSocket(id, starter.SocketName()), + DataDir: m.paths.InstanceDir(id), + GPUProfile: gpuDevice.ProfileName, + GPUFramework: gpuDevice.Framework, + GPUDevicePath: gpuDevice.SysfsPath, + GPUMdevUUID: gpuDevice.MdevUUID, + } + } } }) } @@ -360,7 +381,7 @@ func (m *manager) createInstance( if err != nil { return nil, err } - stored := &StoredMetadata{ + stored = &StoredMetadata{ Id: id, Name: req.Name, Image: req.Image, @@ -610,6 +631,22 @@ func resolveCreateExpiration(req CreateInstanceRequest, now time.Time) (*time.Ti return &expiresAt, nil } +func (m *manager) cleanupFailedCreate(ctx context.Context, id string, retainedVGPU *StoredMetadata) { + if retainedVGPU == nil { + m.deleteInstanceData(id) + return + } + + log := logger.FromContext(ctx) + if err := m.ensureDirectories(id); err != nil { + log.ErrorContext(ctx, "failed to retain instance data after vGPU cleanup failure", "instance_id", id, "error", err) + return + } + if err := m.saveMetadata(&metadata{StoredMetadata: *retainedVGPU}); err != nil { + log.ErrorContext(ctx, "failed to retain vGPU assignment metadata after cleanup failure", "instance_id", id, "error", err) + } +} + // validateCreateRequest validates the create instance request. // The request is mutated in-place to persist normalized egress/credential policy fields. func validateCreateRequest(req *CreateInstanceRequest) error { diff --git a/lib/instances/delete.go b/lib/instances/delete.go index e897ff049..08781b977 100644 --- a/lib/instances/delete.go +++ b/lib/instances/delete.go @@ -150,7 +150,7 @@ func (m *manager) deleteInstanceWithOptions( if hadVGPUAssignment { log.InfoContext(ctx, "destroying vGPU", "instance_id", id, "uuid", stored.GPUMdevUUID) } - if err := releaseStoredVGPU(ctx, stored); err != nil { + if err := m.releaseStoredVGPU(ctx, stored); err != nil { // Log error but continue with cleanup. log.WarnContext(ctx, "failed to destroy vGPU, continuing with cleanup", "instance_id", id, "uuid", stored.GPUMdevUUID, "error", err) } else if hadVGPUAssignment { diff --git a/lib/instances/lifecycle_noop_test.go b/lib/instances/lifecycle_noop_test.go index a9b918dfb..89468d970 100644 --- a/lib/instances/lifecycle_noop_test.go +++ b/lib/instances/lifecycle_noop_test.go @@ -3,6 +3,7 @@ package instances import ( "context" "errors" + "net" "os" "path/filepath" "sync" @@ -195,6 +196,46 @@ func TestDeletePersistsVGPUReleaseBeforeTeardown(t *testing.T) { assert.Equal(t, restartpolicy.BlockedReasonManualStop, persisted.RestartStatus.BlockedReason) } +func TestDeleteDropsStaleVGPUClaimedByLiveInstance(t *testing.T) { + now := time.Now().UTC() + m, id := newLifecycleNoopManagerWithInstance(t, StateStopped, now) + meta, err := m.loadMetadata(id) + require.NoError(t, err) + meta.GPUProfile = "NVIDIA L40S-2Q" + meta.GPUFramework = devices.VGPUFramework("future-framework") + meta.GPUDevicePath = "/sys/bus/pci/devices/0000:82:00.4" + require.NoError(t, m.saveMetadata(meta)) + + claimantID := "inst-live-claimant" + require.NoError(t, m.ensureDirectories(claimantID)) + pid := os.Getpid() + socketPath := m.paths.InstanceSocket(claimantID, "noop.sock") + listener, err := net.Listen("unix", socketPath) + require.NoError(t, err) + defer listener.Close() + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: claimantID, + Name: claimantID, + Image: "test-image", + CreatedAt: now, + HypervisorType: lifecycleNoopHypervisorType, + HypervisorPID: &pid, + SocketPath: socketPath, + DataDir: m.paths.InstanceDir(claimantID), + GPUProfile: "NVIDIA L40S-2Q", + GPUFramework: devices.VGPUFramework("future-framework"), + GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", + }})) + + require.NoError(t, m.DeleteInstance(context.Background(), id)) + + _, err = m.loadMetadata(id) + require.Error(t, err, "deleted instance metadata should be gone") + claimant, err := m.loadMetadata(claimantID) + require.NoError(t, err) + assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", claimant.GPUDevicePath, "live claimant keeps its assignment") +} + func TestDeleteContinuesTeardownAfterFailedVGPURelease(t *testing.T) { m, id := newLifecycleNoopManagerWithInstance(t, StateStopped, time.Now().UTC()) deviceManager := &recordingDeviceManager{} @@ -300,6 +341,20 @@ func (m *recordingDeviceManager) UnbindFromVFIO(ctx context.Context, id string) return nil } +func TestLifecycleNoopStandbyRejectsVendorVFIOVGPU(t *testing.T) { + m, id := newLifecycleNoopManagerWithInstance(t, StateRunning, time.Now().UTC()) + meta, err := m.loadMetadata(id) + require.NoError(t, err) + meta.GPUProfile = "NVIDIA L40S-2Q" + meta.GPUFramework = devices.VGPUFrameworkVendorVFIO + meta.GPUDevicePath = "/sys/bus/pci/devices/0000:82:00.4" + require.NoError(t, m.saveMetadata(meta)) + + _, err = m.StandbyInstance(context.Background(), id, StandbyInstanceRequest{}) + require.ErrorIs(t, err, ErrInvalidState) + assert.ErrorContains(t, err, "standby is not supported for instances with vGPU attached") +} + func newLifecycleNoopManagerWithInstance(t *testing.T, state State, now time.Time) (*manager, string) { t.Helper() diff --git a/lib/instances/start.go b/lib/instances/start.go index a6c832450..ff8e94ea3 100644 --- a/lib/instances/start.go +++ b/lib/instances/start.go @@ -54,7 +54,7 @@ func (m *manager) startInstance( // cannot leave on-disk metadata pointing at a device that is already // gone (matching releaseRetainedVGPULocked). if storedVGPUDevicePath(stored) != "" { - if err := releaseStoredVGPU(ctx, stored); err != nil { + if err := m.releaseStoredVGPU(ctx, stored); err != nil { log.ErrorContext(ctx, "failed to release stale vGPU before start", "instance_id", id, "error", err) return nil, fmt.Errorf("release stale vGPU before start: %w", err) } @@ -181,8 +181,15 @@ func (m *manager) startInstance( } if err := devices.DestroyVGPU(ctx, assignment); err != nil { log.WarnContext(ctx, "failed to destroy vGPU on cleanup", "instance_id", id, "uuid", device.MdevUUID, "error", err) + if saveErr := m.saveMetadata(meta); saveErr != nil { + log.ErrorContext(ctx, "failed to retain vGPU assignment metadata after cleanup failure", "instance_id", id, "error", saveErr) + } } }) + if err := m.saveMetadata(meta); err != nil { + log.ErrorContext(ctx, "failed to save metadata after vGPU creation", "instance_id", id, "error", err) + return nil, fmt.Errorf("save metadata after vGPU creation: %w", err) + } } // 5. Regenerate config disk with new network configuration diff --git a/lib/instances/stop.go b/lib/instances/stop.go index 7eddeb034..ebcf58708 100644 --- a/lib/instances/stop.go +++ b/lib/instances/stop.go @@ -232,7 +232,7 @@ func (m *manager) stopInstance( // 7. Release the vGPU assignment if present (frees the vGPU slot for other VMs). if path := storedVGPUDevicePath(stored); path != "" { log.InfoContext(ctx, "destroying vGPU on stop", "instance_id", id, "device_path", path) - if err := releaseStoredVGPU(ctx, stored); err != nil { + if err := m.releaseStoredVGPU(ctx, stored); err != nil { log.WarnContext(ctx, "failed to destroy vGPU on stop; retaining assignment metadata", "instance_id", id, "device_path", path, "error", err) } } diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index a8ca6aceb..b7d275428 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -20,23 +20,49 @@ func clearStoredVGPUDevice(stored *StoredMetadata) { stored.GPUMdevUUID = "" } -func releaseStoredVGPU(ctx context.Context, stored *StoredMetadata) error { +func (m *manager) releaseStoredVGPU(ctx context.Context, stored *StoredMetadata) error { path := storedVGPUDevicePath(stored) if path != "" { - assignment := devices.VGPUAssignment{ - Framework: stored.GPUFramework, - DevicePath: path, - MdevUUID: stored.GPUMdevUUID, - InstanceID: stored.Id, - } - if err := devices.DestroyVGPU(ctx, assignment); err != nil { + claimed, err := m.vgpuAssignmentClaimedByLiveInstance(ctx, stored.Id, path) + if err != nil { return err } + if claimed { + logger.FromContext(ctx).WarnContext(ctx, "dropping stale vGPU assignment claimed by another live instance", + "instance_id", stored.Id, "device_path", path) + } else { + assignment := devices.VGPUAssignment{ + Framework: stored.GPUFramework, + DevicePath: path, + MdevUUID: stored.GPUMdevUUID, + InstanceID: stored.Id, + } + if err := devices.DestroyVGPU(ctx, assignment); err != nil { + return err + } + } } clearStoredVGPUDevice(stored) return nil } +func (m *manager) vgpuAssignmentClaimedByLiveInstance(ctx context.Context, excludeID, devicePath string) (bool, error) { + instances, err := m.listInstances(ctx) + if err != nil { + return false, fmt.Errorf("list instances for vGPU release check: %w", err) + } + for i := range instances { + inst := &instances[i] + if inst.Id == excludeID || inst.GPUDevicePath != devicePath || inst.HypervisorPID == nil { + continue + } + if HypervisorProcessExists(*inst.HypervisorPID, inst.SocketPath) { + return true, nil + } + } + return false, nil +} + // releaseRetainedVGPULocked releases a vGPU assignment retained on a stopped // instance after a failed release during the original stop. It is a no-op // when no assignment is retained, and a failed retry only logs so the @@ -52,7 +78,7 @@ func (m *manager) releaseRetainedVGPULocked(ctx context.Context, id string) { if storedVGPUDevicePath(stored) == "" { return } - if err := releaseStoredVGPU(ctx, stored); err != nil { + if err := m.releaseStoredVGPU(ctx, stored); err != nil { log.WarnContext(ctx, "failed to destroy retained vGPU; retaining assignment metadata", "instance_id", id, "error", err) return } diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index 6f2c46819..2b7d84a94 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -5,14 +5,47 @@ import ( "testing" "github.com/kernel/hypeman/lib/devices" + "github.com/kernel/hypeman/lib/hypervisor" + "github.com/kernel/hypeman/lib/paths" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) +func TestValidateVGPUHypervisor(t *testing.T) { + t.Parallel() + + assert.NoError(t, validateVGPUHypervisor(hypervisor.TypeQEMU)) + assert.EqualError(t, validateVGPUHypervisor(hypervisor.TypeCloudHypervisor), "vGPU is only supported with qemu, got cloud-hypervisor") +} + +func TestCleanupFailedCreateRetainsVGPUAssignment(t *testing.T) { + t.Parallel() + + m := &manager{paths: paths.New(t.TempDir())} + stored := &StoredMetadata{ + Id: "failed-create", + Name: "failed-create", + GPUProfile: "NVIDIA L40S-2Q", + GPUFramework: devices.VGPUFrameworkVendorVFIO, + GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", + HypervisorType: "qemu", + DataDir: m.paths.InstanceDir("failed-create"), + } + + m.cleanupFailedCreate(context.Background(), stored.Id, stored) + + retained, err := m.loadMetadata(stored.Id) + require.NoError(t, err) + assert.Equal(t, stored.GPUProfile, retained.GPUProfile) + assert.Equal(t, stored.GPUFramework, retained.GPUFramework) + assert.Equal(t, stored.GPUDevicePath, retained.GPUDevicePath) +} + func TestStoredVGPUDevicePath(t *testing.T) { t.Parallel() - assert.Equal(t, "/sys/bus/mdev/devices/new-uuid", storedVGPUDevicePath(&StoredMetadata{ - GPUDevicePath: "/sys/bus/mdev/devices/new-uuid", + assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", storedVGPUDevicePath(&StoredMetadata{ + GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", GPUMdevUUID: "legacy-uuid", })) assert.Equal(t, "/sys/bus/mdev/devices/legacy-uuid", storedVGPUDevicePath(&StoredMetadata{ @@ -24,11 +57,12 @@ func TestStoredVGPUDevicePath(t *testing.T) { func TestReleaseStoredVGPURetainsMetadataOnFailure(t *testing.T) { t.Parallel() + m := &manager{paths: paths.New(t.TempDir())} stored := &StoredMetadata{ GPUFramework: devices.VGPUFramework("future-framework"), GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", } - err := releaseStoredVGPU(context.Background(), stored) + err := m.releaseStoredVGPU(context.Background(), stored) assert.Error(t, err) assert.Equal(t, devices.VGPUFramework("future-framework"), stored.GPUFramework) assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", stored.GPUDevicePath) @@ -39,13 +73,11 @@ func TestSetAndClearStoredVGPUDevice(t *testing.T) { stored := &StoredMetadata{} setStoredVGPUDevice(stored, &devices.VGPUDevice{ - Framework: devices.VGPUFrameworkMdev, - SysfsPath: "/sys/bus/mdev/devices/new-uuid", - MdevUUID: "new-uuid", + Framework: devices.VGPUFrameworkVendorVFIO, + SysfsPath: "/sys/bus/pci/devices/0000:82:00.4", }) - assert.Equal(t, devices.VGPUFrameworkMdev, stored.GPUFramework) - assert.Equal(t, "/sys/bus/mdev/devices/new-uuid", stored.GPUDevicePath) - assert.Equal(t, "new-uuid", stored.GPUMdevUUID) + assert.Equal(t, devices.VGPUFrameworkVendorVFIO, stored.GPUFramework) + assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", stored.GPUDevicePath) clearStoredVGPUDevice(stored) assert.Empty(t, stored.GPUFramework) From 5e7955a5cca4cce96280a640cdf66c2595152971 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Thu, 6 Aug 2026 19:05:42 +0000 Subject: [PATCH 02/76] Reconcile vendor VFIO vGPUs against a fail-closed instance inventory Startup reconciliation protects the VFs of instances whose hypervisor survived the restart, verified by socket ownership so a reused PID cannot hold a VF. The inventory behind that protected set must not silently skip unreadable metadata: a skipped live claimant would leave its VF unprotected during the pre-VFIO-open boot window. Add ListInstancesForReconcile, which fails on any unreadable metadata, and skip vendor VFIO reconciliation when the inventory is unavailable while keeping mdev reconciliation running. --- cmd/api/main.go | 33 ++++++++++++++++++++++++++++----- lib/builds/manager_test.go | 4 ++++ lib/instances/manager.go | 6 ++++++ lib/instances/query.go | 13 +++++++++++-- lib/instances/query_test.go | 22 ++++++++++++++++++++++ lib/instances/storage.go | 8 +++++++- lib/instances/wait_test.go | 3 +++ 7 files changed, 81 insertions(+), 8 deletions(-) diff --git a/cmd/api/main.go b/cmd/api/main.go index f1bbfcf3d..0479f1583 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -185,6 +185,24 @@ func configureUFFDGraduationController(cfg *config.Config, instanceManager insta }, logger), nil } +func liveInstanceVGPUDevicePaths(ctx context.Context, instanceManager instances.Manager) (map[string]struct{}, error) { + allInstances, err := instanceManager.ListInstancesForReconcile(ctx) + if err != nil { + return nil, err + } + protected := make(map[string]struct{}) + for _, inst := range allInstances { + if inst.GPUDevicePath == "" || inst.HypervisorPID == nil { + continue + } + if !instances.HypervisorProcessExists(*inst.HypervisorPID, inst.SocketPath) { + continue + } + protected[inst.GPUDevicePath] = struct{}{} + } + return protected, nil +} + func run() error { startupStarted := time.Now() slog.Info("starting hypeman initialization") @@ -384,11 +402,16 @@ func run() error { return fmt.Errorf("reconcile device state: %w", err) } - // Reconcile mdev devices (clears orphaned vGPUs from previous runs) - logger.Info("Reconciling mdev devices...") - if err := devices.ReconcileMdevs(app.Ctx, nil); err != nil { - // Log but don't fail - mdev cleanup is best-effort - logger.Warn("failed to reconcile mdev devices", "error", err) + // Reconcile vGPU devices (clears orphaned vGPUs from previous runs) + logger.Info("Reconciling vGPU devices...") + protected, err := liveInstanceVGPUDevicePaths(app.Ctx, app.InstanceManager) + if err != nil { + logger.Warn("failed to list instances for vGPU reconcile protection; skipping vendor VFIO reconciliation", "error", err) + protected = nil + } + if err := devices.ReconcileVGPUs(app.Ctx, protected); err != nil { + // Log but don't fail - vGPU cleanup is best-effort + logger.Warn("failed to reconcile vGPU devices", "error", err) } // Wire up resource validator for aggregate limit checking diff --git a/lib/builds/manager_test.go b/lib/builds/manager_test.go index a137edc66..44596bf68 100644 --- a/lib/builds/manager_test.go +++ b/lib/builds/manager_test.go @@ -51,6 +51,10 @@ func (m *mockInstanceManager) ListInstances(ctx context.Context, filter *instanc return result, nil } +func (m *mockInstanceManager) ListInstancesForReconcile(ctx context.Context) ([]instances.Instance, error) { + return m.ListInstances(ctx, nil) +} + func (m *mockInstanceManager) ListSnapshots(ctx context.Context, filter *instances.ListSnapshotsFilter) ([]instances.Snapshot, error) { return nil, nil } diff --git a/lib/instances/manager.go b/lib/instances/manager.go index bc23fdf74..dd4404972 100644 --- a/lib/instances/manager.go +++ b/lib/instances/manager.go @@ -27,6 +27,7 @@ import ( type Manager interface { ListInstances(ctx context.Context, filter *ListInstancesFilter) ([]Instance, error) + ListInstancesForReconcile(ctx context.Context) ([]Instance, error) ListSnapshots(ctx context.Context, filter *ListSnapshotsFilter) ([]Snapshot, error) GetSnapshot(ctx context.Context, snapshotID string) (*Snapshot, error) CreateInstance(ctx context.Context, req CreateInstanceRequest) (*Instance, error) @@ -732,6 +733,11 @@ func (m *manager) DefaultHypervisor() hypervisor.Type { return m.defaultHypervisor } +// ListInstancesForReconcile returns every instance or an invalid metadata error. +func (m *manager) ListInstancesForReconcile(ctx context.Context) ([]Instance, error) { + return m.loadInstances(ctx, false) +} + // ListInstances returns instances, optionally filtered by the given criteria. // Pass nil to return all instances. func (m *manager) ListInstances(ctx context.Context, filter *ListInstancesFilter) ([]Instance, error) { diff --git a/lib/instances/query.go b/lib/instances/query.go index 8e3ed61f1..eea66ab13 100644 --- a/lib/instances/query.go +++ b/lib/instances/query.go @@ -784,14 +784,18 @@ func parseSentinelTimestamp(line, sentinelPrefix string) (time.Time, bool) { return time.Time{}, false } -// listInstances returns all instances +// listInstances returns all instances, skipping metadata files that cannot be loaded. func (m *manager) listInstances(ctx context.Context) ([]Instance, error) { + return m.loadInstances(ctx, true) +} + +func (m *manager) loadInstances(ctx context.Context, skipInvalid bool) ([]Instance, error) { ctx, span := m.tracerOrDefault().Start(ctx, "instances.list_metadata") defer span.End() log := logger.FromContext(ctx) log.DebugContext(ctx, "listing all instances") - files, err := m.listMetadataFiles() + files, err := m.listMetadataFilesWithStatErrors(!skipInvalid) if err != nil { log.ErrorContext(ctx, "failed to list metadata files", "error", err) return nil, err @@ -809,6 +813,11 @@ func (m *manager) listInstances(ctx context.Context) ([]Instance, error) { ) meta, err := m.loadMetadata(id) if err != nil { + if !skipInvalid { + hydrateSpan.RecordError(err) + hydrateSpan.End() + return nil, fmt.Errorf("load metadata for instance %s: %w", id, err) + } // Skip instances with invalid metadata log.WarnContext(hydrateCtx, "skipping instance with invalid metadata", "instance_id", id, "error", err) hydrateSpan.End() diff --git a/lib/instances/query_test.go b/lib/instances/query_test.go index 8bb0bb464..41aba54e8 100644 --- a/lib/instances/query_test.go +++ b/lib/instances/query_test.go @@ -14,6 +14,28 @@ import ( "github.com/stretchr/testify/require" ) +func TestListInstancesForReconcileFailsOnInvalidMetadata(t *testing.T) { + m := &manager{paths: paths.New(t.TempDir())} + + require.NoError(t, m.ensureDirectories("valid")) + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: "valid", + Name: "valid", + CreatedAt: time.Now(), + DataDir: m.paths.InstanceDir("valid"), + }})) + require.NoError(t, m.ensureDirectories("invalid")) + require.NoError(t, os.WriteFile(m.paths.InstanceMetadata("invalid"), []byte("{"), 0644)) + + listed, err := m.ListInstances(context.Background(), nil) + require.NoError(t, err) + require.Len(t, listed, 1) + + _, err = m.ListInstancesForReconcile(context.Background()) + require.Error(t, err) + assert.ErrorContains(t, err, "load metadata for instance invalid") +} + func TestParseExitSentinelLine(t *testing.T) { t.Parallel() tests := []struct { diff --git a/lib/instances/storage.go b/lib/instances/storage.go index a293fc6e1..dd932d41a 100644 --- a/lib/instances/storage.go +++ b/lib/instances/storage.go @@ -187,8 +187,12 @@ func removeAllWithRetry(path string, removeAll func(string) error, sleep func(ti } } -// listMetadataFiles returns paths to all instance metadata files +// listMetadataFiles returns paths to all instance metadata files. func (m *manager) listMetadataFiles() ([]string, error) { + return m.listMetadataFilesWithStatErrors(false) +} + +func (m *manager) listMetadataFilesWithStatErrors(failOnStatError bool) ([]string, error) { guestsDir := m.paths.GuestsDir() // Ensure guests directory exists @@ -210,6 +214,8 @@ func (m *manager) listMetadataFiles() ([]string, error) { metaPath := filepath.Join(guestsDir, entry.Name(), "metadata.json") if _, err := os.Stat(metaPath); err == nil { metaFiles = append(metaFiles, metaPath) + } else if failOnStatError && !os.IsNotExist(err) { + return nil, fmt.Errorf("stat metadata for instance %s: %w", entry.Name(), err) } } diff --git a/lib/instances/wait_test.go b/lib/instances/wait_test.go index dbb630185..415003594 100644 --- a/lib/instances/wait_test.go +++ b/lib/instances/wait_test.go @@ -32,6 +32,9 @@ func (s *stubManager) GetInstance(ctx context.Context, id string) (*Instance, er func (s *stubManager) ListInstances(context.Context, *ListInstancesFilter) ([]Instance, error) { return nil, nil } +func (s *stubManager) ListInstancesForReconcile(context.Context) ([]Instance, error) { + return nil, nil +} func (s *stubManager) ListSnapshots(context.Context, *ListSnapshotsFilter) ([]Snapshot, error) { return nil, nil } From d7eadf8db0e0ac8dd6a7949191909d0b013c120c Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Thu, 6 Aug 2026 20:19:26 +0000 Subject: [PATCH 03/76] Fail closed on vGPU claim checks --- lib/instances/vgpu.go | 2 +- lib/instances/vgpu_test.go | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index b7d275428..29edfeab7 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -47,7 +47,7 @@ func (m *manager) releaseStoredVGPU(ctx context.Context, stored *StoredMetadata) } func (m *manager) vgpuAssignmentClaimedByLiveInstance(ctx context.Context, excludeID, devicePath string) (bool, error) { - instances, err := m.listInstances(ctx) + instances, err := m.ListInstancesForReconcile(ctx) if err != nil { return false, fmt.Errorf("list instances for vGPU release check: %w", err) } diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index 2b7d84a94..29341d230 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -2,6 +2,7 @@ package instances import ( "context" + "os" "testing" "github.com/kernel/hypeman/lib/devices" @@ -41,6 +42,17 @@ func TestCleanupFailedCreateRetainsVGPUAssignment(t *testing.T) { assert.Equal(t, stored.GPUDevicePath, retained.GPUDevicePath) } +func TestVGPUAssignmentClaimedByLiveInstanceFailsOnInvalidMetadata(t *testing.T) { + t.Parallel() + + m := &manager{paths: paths.New(t.TempDir())} + require.NoError(t, m.ensureDirectories("invalid-instance")) + require.NoError(t, os.WriteFile(m.paths.InstanceMetadata("invalid-instance"), []byte("{"), 0o644)) + + _, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "other-instance", "/sys/bus/pci/devices/0000:82:00.4") + require.Error(t, err) +} + func TestStoredVGPUDevicePath(t *testing.T) { t.Parallel() From 506b42117e168ac014f3b98fd92c88ec721c0d4d Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Thu, 6 Aug 2026 20:19:50 +0000 Subject: [PATCH 04/76] Retain only vGPU assignment after failed create --- lib/instances/create.go | 8 +++++++- lib/instances/vgpu_test.go | 13 ++++++++++++- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/lib/instances/create.go b/lib/instances/create.go index 2608b2bd2..d25bd91ed 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -642,7 +642,13 @@ func (m *manager) cleanupFailedCreate(ctx context.Context, id string, retainedVG log.ErrorContext(ctx, "failed to retain instance data after vGPU cleanup failure", "instance_id", id, "error", err) return } - if err := m.saveMetadata(&metadata{StoredMetadata: *retainedVGPU}); err != nil { + retained := StoredMetadata{ + Id: id, + GPUFramework: retainedVGPU.GPUFramework, + GPUDevicePath: retainedVGPU.GPUDevicePath, + GPUMdevUUID: retainedVGPU.GPUMdevUUID, + } + if err := m.saveMetadata(&metadata{StoredMetadata: retained}); err != nil { log.ErrorContext(ctx, "failed to retain vGPU assignment metadata after cleanup failure", "instance_id", id, "error", err) } } diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index 29341d230..b408ccafe 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -29,6 +29,10 @@ func TestCleanupFailedCreateRetainsVGPUAssignment(t *testing.T) { GPUProfile: "NVIDIA L40S-2Q", GPUFramework: devices.VGPUFrameworkVendorVFIO, GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", + GPUMdevUUID: "mdev-uuid", + NetworkEnabled: true, + IP: "192.0.2.1", + Volumes: []VolumeAttachment{{VolumeID: "volume"}}, HypervisorType: "qemu", DataDir: m.paths.InstanceDir("failed-create"), } @@ -37,9 +41,16 @@ func TestCleanupFailedCreateRetainsVGPUAssignment(t *testing.T) { retained, err := m.loadMetadata(stored.Id) require.NoError(t, err) - assert.Equal(t, stored.GPUProfile, retained.GPUProfile) + assert.Equal(t, stored.Id, retained.Id) assert.Equal(t, stored.GPUFramework, retained.GPUFramework) assert.Equal(t, stored.GPUDevicePath, retained.GPUDevicePath) + assert.Equal(t, stored.GPUMdevUUID, retained.GPUMdevUUID) + assert.Empty(t, retained.Name) + assert.Empty(t, retained.GPUProfile) + assert.False(t, retained.NetworkEnabled) + assert.Empty(t, retained.IP) + assert.Empty(t, retained.Volumes) + assert.Empty(t, retained.DataDir) } func TestVGPUAssignmentClaimedByLiveInstanceFailsOnInvalidMetadata(t *testing.T) { From bd1fd3898180d48f0757d91a24a5f87bfe65c30e Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Thu, 6 Aug 2026 20:20:05 +0000 Subject: [PATCH 05/76] Clear released vGPU assignment on start rollback --- lib/instances/start.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/instances/start.go b/lib/instances/start.go index ff8e94ea3..d9c8c9aaa 100644 --- a/lib/instances/start.go +++ b/lib/instances/start.go @@ -184,6 +184,11 @@ func (m *manager) startInstance( if saveErr := m.saveMetadata(meta); saveErr != nil { log.ErrorContext(ctx, "failed to retain vGPU assignment metadata after cleanup failure", "instance_id", id, "error", saveErr) } + } else { + clearStoredVGPUDevice(stored) + if saveErr := m.saveMetadata(meta); saveErr != nil { + log.ErrorContext(ctx, "failed to save metadata after vGPU cleanup", "instance_id", id, "error", saveErr) + } } }) if err := m.saveMetadata(meta); err != nil { From 5ca85a2a6fb65f97fac6b17a028ed840239eead9 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Thu, 6 Aug 2026 20:50:00 +0000 Subject: [PATCH 06/76] Test start rollback vGPU cleanup --- lib/instances/vgpu_test.go | 70 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index b408ccafe..d9aa02f02 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -3,7 +3,10 @@ package instances import ( "context" "os" + "path/filepath" + "sync" "testing" + _ "unsafe" "github.com/kernel/hypeman/lib/devices" "github.com/kernel/hypeman/lib/hypervisor" @@ -53,6 +56,73 @@ func TestCleanupFailedCreateRetainsVGPUAssignment(t *testing.T) { assert.Empty(t, retained.DataDir) } +//go:linkname hostVendorVFIO github.com/kernel/hypeman/lib/devices.hostVendorVFIO +var hostVendorVFIO vendorVFIOSysfs + +type vendorVFIOSysfs struct { + pciDevicesPath string + procPath string + vfioDevicesPath string + owners map[string]string +} + +func TestStartRollbackClearsVGPUAssignmentAfterSuccessfulDestroy(t *testing.T) { + root := t.TempDir() + pciDevicesPath := filepath.Join(root, "sys", "bus", "pci", "devices") + vfAddress := "0000:82:00.4" + nvidiaPath := filepath.Join(pciDevicesPath, vfAddress, "nvidia") + require.NoError(t, os.MkdirAll(nvidiaPath, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(nvidiaPath, "current_vgpu_type"), []byte("0"), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(nvidiaPath, "creatable_vgpu_types"), []byte("ID : vGPU Name\n1148 : NVIDIA L40S-2Q\n"), 0o644)) + + originalVendorVFIO := hostVendorVFIO + hostVendorVFIO = vendorVFIOSysfs{ + pciDevicesPath: pciDevicesPath, + procPath: filepath.Join(root, "proc"), + vfioDevicesPath: filepath.Join(root, "dev", "vfio", "devices"), + owners: make(map[string]string), + } + t.Cleanup(func() { hostVendorVFIO = originalVendorVFIO }) + require.NoError(t, os.MkdirAll(hostVendorVFIO.procPath, 0o755)) + + m := &manager{ + paths: paths.New(t.TempDir()), + imageManager: readyFixtureImageManager{name: "test-image"}, + instanceLocks: sync.Map{}, + bootMarkerScans: sync.Map{}, + } + const id = "start-rollback" + require.NoError(t, m.ensureDirectories(id)) + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: id, + Name: id, + Image: "test-image", + GPUProfile: "NVIDIA L40S-2Q", + HypervisorType: lifecycleNoopHypervisorType, + SocketPath: m.paths.InstanceSocket(id, "noop.sock"), + DataDir: m.paths.InstanceDir(id), + }})) + + t.Setenv("TMPDIR", filepath.Join(root, "missing")) + _, err := m.startInstance(context.Background(), id, StartInstanceRequest{}) + require.Error(t, err) + + stored, err := m.loadMetadata(id) + require.NoError(t, err) + assert.Equal(t, "NVIDIA L40S-2Q", stored.GPUProfile) + assert.Empty(t, stored.GPUFramework) + assert.Empty(t, stored.GPUDevicePath) + assert.Empty(t, stored.GPUMdevUUID) + assertFileContents(t, filepath.Join(nvidiaPath, "current_vgpu_type"), "0") +} + +func assertFileContents(t *testing.T, path, want string) { + t.Helper() + got, err := os.ReadFile(path) + require.NoError(t, err) + assert.Equal(t, want, string(got)) +} + func TestVGPUAssignmentClaimedByLiveInstanceFailsOnInvalidMetadata(t *testing.T) { t.Parallel() From afc8dade88d6c3f9d5cb58a3c23e996a2dd93c38 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Fri, 7 Aug 2026 15:02:16 +0000 Subject: [PATCH 07/76] Normalize legacy mdev paths in live-claim check The claim guard compared raw GPUDevicePath, which is empty on records persisted before the framework migration; a live claimant with only a legacy GPUMdevUUID was invisible to the check. Normalize the inventory side with storedVGPUDevicePath, matching the release subject. --- lib/instances/vgpu.go | 2 +- lib/instances/vgpu_test.go | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index 29edfeab7..23ea2d825 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -53,7 +53,7 @@ func (m *manager) vgpuAssignmentClaimedByLiveInstance(ctx context.Context, exclu } for i := range instances { inst := &instances[i] - if inst.Id == excludeID || inst.GPUDevicePath != devicePath || inst.HypervisorPID == nil { + if inst.Id == excludeID || storedVGPUDevicePath(&inst.StoredMetadata) != devicePath || inst.HypervisorPID == nil { continue } if HypervisorProcessExists(*inst.HypervisorPID, inst.SocketPath) { diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index d9aa02f02..79e6fe0c0 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -134,6 +134,24 @@ func TestVGPUAssignmentClaimedByLiveInstanceFailsOnInvalidMetadata(t *testing.T) require.Error(t, err) } +func TestVGPUAssignmentClaimedByLiveInstanceNormalizesLegacyMdevPath(t *testing.T) { + t.Parallel() + + m := &manager{paths: paths.New(t.TempDir())} + require.NoError(t, m.ensureDirectories("legacy-claimant")) + pid := os.Getpid() + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: "legacy-claimant", + Name: "legacy-claimant", + GPUMdevUUID: "legacy-uuid", + HypervisorPID: &pid, + }})) + + claimed, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "other-instance", "/sys/bus/mdev/devices/legacy-uuid") + require.NoError(t, err) + assert.True(t, claimed) +} + func TestStoredVGPUDevicePath(t *testing.T) { t.Parallel() From df1a288748c7141f9f1a4f00100fa99b727faab5 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Fri, 7 Aug 2026 20:45:49 +0000 Subject: [PATCH 08/76] Bind the live-claimant test socket under /tmp for macOS --- lib/instances/lifecycle_noop_test.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/lib/instances/lifecycle_noop_test.go b/lib/instances/lifecycle_noop_test.go index 89468d970..f3baa9dae 100644 --- a/lib/instances/lifecycle_noop_test.go +++ b/lib/instances/lifecycle_noop_test.go @@ -209,7 +209,14 @@ func TestDeleteDropsStaleVGPUClaimedByLiveInstance(t *testing.T) { claimantID := "inst-live-claimant" require.NoError(t, m.ensureDirectories(claimantID)) pid := os.Getpid() - socketPath := m.paths.InstanceSocket(claimantID, "noop.sock") + // Bind under /tmp: a t.TempDir()-derived path exceeds the macOS AF_UNIX + // path limit. + socketDir, err := os.MkdirTemp("/tmp", "hypeman-claimant-socket-") + require.NoError(t, err) + t.Cleanup(func() { + _ = os.RemoveAll(socketDir) + }) + socketPath := filepath.Join(socketDir, "noop.sock") listener, err := net.Listen("unix", socketPath) require.NoError(t, err) defer listener.Close() From 44536513a33b084c32558a4b9409f68c18f32e18 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Fri, 7 Aug 2026 20:45:49 +0000 Subject: [PATCH 09/76] Surface retained vGPU cleanup through a typed create error and manager seam Replace the go:linkname shadow of devices.hostVendorVFIO with createVGPU/destroyVGPU manager fields, and wrap failed creates whose rollback release also failed in VGPUCleanupPendingError so the API can point callers at the retained instance record. --- cmd/api/api/instances.go | 7 +++ lib/instances/create.go | 28 +++++++++--- lib/instances/manager.go | 4 ++ lib/instances/start.go | 8 ++-- lib/instances/vgpu.go | 40 ++++++++++++++++- lib/instances/vgpu_test.go | 92 ++++++++++++++++++++++++-------------- 6 files changed, 134 insertions(+), 45 deletions(-) diff --git a/cmd/api/api/instances.go b/cmd/api/api/instances.go index 669d38633..e9cee6ed9 100644 --- a/cmd/api/api/instances.go +++ b/cmd/api/api/instances.go @@ -362,6 +362,7 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst inst, err := s.InstanceManager.CreateInstance(ctx, domainReq) if err != nil { + var vgpuPending *instances.VGPUCleanupPendingError switch { case errors.Is(err, instances.ErrImageNotReady): return oapi.CreateInstance400JSONResponse{ @@ -413,6 +414,12 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst Code: "not_found", Message: err.Error(), }, nil + case errors.As(err, &vgpuPending): + log.ErrorContext(ctx, "failed to create instance", "error", err, "image", request.Body.Image) + return oapi.CreateInstance500JSONResponse{ + Code: "vgpu_cleanup_pending", + Message: fmt.Sprintf("failed to create instance; vGPU release failed during rollback and instance %s retains the assignment, delete it to retry", vgpuPending.InstanceID), + }, nil default: log.ErrorContext(ctx, "failed to create instance", "error", err, "image", request.Body.Image) return oapi.CreateInstance500JSONResponse{ diff --git a/lib/instances/create.go b/lib/instances/create.go index d25bd91ed..dc70639fd 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -280,10 +280,20 @@ func (m *manager) createInstance( var stored *StoredMetadata var retainedVGPU *StoredMetadata - // Setup cleanup stack early so device attachment errors trigger cleanup + // Setup cleanup stack early so device attachment errors trigger cleanup. + // When rollback retains a vGPU assignment, surface the retained instance + // ID to the caller so the record is discoverable and can be deleted to + // retry the release. The wrapping defer is registered first so it runs + // after cu.Clean has decided whether metadata was retained. + vgpuRetained := false + defer func() { + if retErr != nil && vgpuRetained { + retErr = &VGPUCleanupPendingError{InstanceID: id, Err: retErr} + } + }() cu := cleanup.Make(func() { log.DebugContext(ctx, "cleaning up instance on error", "instance_id", id) - m.cleanupFailedCreate(ctx, id, retainedVGPU) + vgpuRetained = m.cleanupFailedCreate(ctx, id, retainedVGPU) }) defer cu.Clean() @@ -300,7 +310,7 @@ func (m *manager) createInstance( // Handle vGPU profile request if req.GPU != nil && req.GPU.Profile != "" { log.InfoContext(ctx, "creating vGPU", "instance_id", id, "profile", req.GPU.Profile) - gpuDevice, err = devices.CreateVGPU(ctx, req.GPU.Profile, id) + gpuDevice, err = m.createVGPUDevice(ctx, req.GPU.Profile, id) if err != nil { log.ErrorContext(ctx, "failed to create vGPU", "profile", req.GPU.Profile, "error", err) return nil, wrapCreateVGPUErr(req.GPU.Profile, err) @@ -320,7 +330,7 @@ func (m *manager) createInstance( MdevUUID: gpuDevice.MdevUUID, InstanceID: id, } - if err := devices.DestroyVGPU(ctx, assignment); err != nil { + if err := m.destroyVGPUAssignment(ctx, assignment); err != nil { log.WarnContext(ctx, "failed to destroy vGPU on cleanup", "instance_id", id, "uuid", gpuDevice.MdevUUID, "error", err) retainedVGPU = stored if retainedVGPU == nil { @@ -631,16 +641,18 @@ func resolveCreateExpiration(req CreateInstanceRequest, now time.Time) (*time.Ti return &expiresAt, nil } -func (m *manager) cleanupFailedCreate(ctx context.Context, id string, retainedVGPU *StoredMetadata) { +// cleanupFailedCreate reports whether it retained instance metadata for a +// vGPU assignment whose release failed during rollback. +func (m *manager) cleanupFailedCreate(ctx context.Context, id string, retainedVGPU *StoredMetadata) bool { if retainedVGPU == nil { m.deleteInstanceData(id) - return + return false } log := logger.FromContext(ctx) if err := m.ensureDirectories(id); err != nil { log.ErrorContext(ctx, "failed to retain instance data after vGPU cleanup failure", "instance_id", id, "error", err) - return + return false } retained := StoredMetadata{ Id: id, @@ -650,7 +662,9 @@ func (m *manager) cleanupFailedCreate(ctx context.Context, id string, retainedVG } if err := m.saveMetadata(&metadata{StoredMetadata: retained}); err != nil { log.ErrorContext(ctx, "failed to retain vGPU assignment metadata after cleanup failure", "instance_id", id, "error", err) + return false } + return true } // validateCreateRequest validates the create instance request. diff --git a/lib/instances/manager.go b/lib/instances/manager.go index dd4404972..458275bb5 100644 --- a/lib/instances/manager.go +++ b/lib/instances/manager.go @@ -182,6 +182,8 @@ type manager struct { now func() time.Time writeFile func(string, []byte, os.FileMode) error deleteInstanceFn func(context.Context, string) error + createVGPU func(context.Context, string, string) (*devices.VGPUDevice, error) + destroyVGPU func(context.Context, devices.VGPUAssignment) error deleteSnapshotFn func(context.Context, string) error ttlReaperDeleteTimeout time.Duration egressProxy *egressproxy.Service @@ -282,6 +284,8 @@ func NewManagerWithConfigE(p *paths.Paths, imageManager images.Manager, systemMa defaultHypervisor: defaultHypervisor, now: time.Now, writeFile: os.WriteFile, + createVGPU: devices.CreateVGPU, + destroyVGPU: devices.DestroyVGPU, meter: meter, tracer: tracer, guestMemoryPolicy: policy, diff --git a/lib/instances/start.go b/lib/instances/start.go index d9c8c9aaa..b830bafb2 100644 --- a/lib/instances/start.go +++ b/lib/instances/start.go @@ -162,11 +162,11 @@ func (m *manager) startInstance( // 4b. Recreate the vGPU if this instance had a GPU profile // Note: GPU availability was already validated in step 2b if stored.GPUProfile != "" { - log.InfoContext(ctx, "creating vGPU mdev for start", "instance_id", id, "profile", stored.GPUProfile) - device, err := devices.CreateVGPU(ctx, stored.GPUProfile, id) + log.InfoContext(ctx, "creating vGPU for start", "instance_id", id, "profile", stored.GPUProfile) + device, err := m.createVGPUDevice(ctx, stored.GPUProfile, id) if err != nil { log.ErrorContext(ctx, "failed to create vGPU", "instance_id", id, "profile", stored.GPUProfile, "error", err) - return nil, fmt.Errorf("create vGPU mdev for profile %s: %w", stored.GPUProfile, err) + return nil, fmt.Errorf("create vGPU for profile %s: %w", stored.GPUProfile, err) } setStoredVGPUDevice(stored, device) log.InfoContext(ctx, "created vGPU", "instance_id", id, "profile", stored.GPUProfile, "uuid", device.MdevUUID) @@ -179,7 +179,7 @@ func (m *manager) startInstance( MdevUUID: device.MdevUUID, InstanceID: id, } - if err := devices.DestroyVGPU(ctx, assignment); err != nil { + if err := m.destroyVGPUAssignment(ctx, assignment); err != nil { log.WarnContext(ctx, "failed to destroy vGPU on cleanup", "instance_id", id, "uuid", device.MdevUUID, "error", err) if saveErr := m.saveMetadata(meta); saveErr != nil { log.ErrorContext(ctx, "failed to retain vGPU assignment metadata after cleanup failure", "instance_id", id, "error", saveErr) diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index 23ea2d825..ae100a39e 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -5,9 +5,47 @@ import ( "path/filepath" "github.com/kernel/hypeman/lib/devices" + "github.com/kernel/hypeman/lib/hypervisor" "github.com/kernel/hypeman/lib/logger" ) +func validateVGPUHypervisor(hvType hypervisor.Type) error { + if hvType != hypervisor.TypeQEMU { + return fmt.Errorf("vGPU is only supported with qemu, got %s", hvType) + } + return nil +} + +// VGPUCleanupPendingError reports a failed create whose vGPU release also +// failed during rollback. The instance record identified by InstanceID is +// retained so the release can be retried; deleting the instance retries it. +type VGPUCleanupPendingError struct { + InstanceID string + Err error +} + +func (e *VGPUCleanupPendingError) Error() string { + return fmt.Sprintf("%v; vGPU release failed during rollback, instance %s retains the assignment", e.Err, e.InstanceID) +} + +func (e *VGPUCleanupPendingError) Unwrap() error { return e.Err } + +func (m *manager) createVGPUDevice(ctx context.Context, profileName, instanceID string) (*devices.VGPUDevice, error) { + create := m.createVGPU + if create == nil { + create = devices.CreateVGPU + } + return create(ctx, profileName, instanceID) +} + +func (m *manager) destroyVGPUAssignment(ctx context.Context, assignment devices.VGPUAssignment) error { + destroy := m.destroyVGPU + if destroy == nil { + destroy = devices.DestroyVGPU + } + return destroy(ctx, assignment) +} + func setStoredVGPUDevice(stored *StoredMetadata, device *devices.VGPUDevice) { stored.GPUFramework = device.Framework stored.GPUDevicePath = device.SysfsPath @@ -37,7 +75,7 @@ func (m *manager) releaseStoredVGPU(ctx context.Context, stored *StoredMetadata) MdevUUID: stored.GPUMdevUUID, InstanceID: stored.Id, } - if err := devices.DestroyVGPU(ctx, assignment); err != nil { + if err := m.destroyVGPUAssignment(ctx, assignment); err != nil { return err } } diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index 79e6fe0c0..da8426efe 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -2,11 +2,11 @@ package instances import ( "context" + "errors" "os" "path/filepath" "sync" "testing" - _ "unsafe" "github.com/kernel/hypeman/lib/devices" "github.com/kernel/hypeman/lib/hypervisor" @@ -40,7 +40,7 @@ func TestCleanupFailedCreateRetainsVGPUAssignment(t *testing.T) { DataDir: m.paths.InstanceDir("failed-create"), } - m.cleanupFailedCreate(context.Background(), stored.Id, stored) + assert.True(t, m.cleanupFailedCreate(context.Background(), stored.Id, stored)) retained, err := m.loadMetadata(stored.Id) require.NoError(t, err) @@ -56,40 +56,43 @@ func TestCleanupFailedCreateRetainsVGPUAssignment(t *testing.T) { assert.Empty(t, retained.DataDir) } -//go:linkname hostVendorVFIO github.com/kernel/hypeman/lib/devices.hostVendorVFIO -var hostVendorVFIO vendorVFIOSysfs +func TestCleanupFailedCreateDeletesDataWithoutRetainedVGPU(t *testing.T) { + t.Parallel() + + m := &manager{paths: paths.New(t.TempDir())} + require.NoError(t, m.ensureDirectories("failed-create")) -type vendorVFIOSysfs struct { - pciDevicesPath string - procPath string - vfioDevicesPath string - owners map[string]string + assert.False(t, m.cleanupFailedCreate(context.Background(), "failed-create", nil)) + _, err := m.loadMetadata("failed-create") + require.Error(t, err) } -func TestStartRollbackClearsVGPUAssignmentAfterSuccessfulDestroy(t *testing.T) { - root := t.TempDir() - pciDevicesPath := filepath.Join(root, "sys", "bus", "pci", "devices") - vfAddress := "0000:82:00.4" - nvidiaPath := filepath.Join(pciDevicesPath, vfAddress, "nvidia") - require.NoError(t, os.MkdirAll(nvidiaPath, 0o755)) - require.NoError(t, os.WriteFile(filepath.Join(nvidiaPath, "current_vgpu_type"), []byte("0"), 0o644)) - require.NoError(t, os.WriteFile(filepath.Join(nvidiaPath, "creatable_vgpu_types"), []byte("ID : vGPU Name\n1148 : NVIDIA L40S-2Q\n"), 0o644)) - - originalVendorVFIO := hostVendorVFIO - hostVendorVFIO = vendorVFIOSysfs{ - pciDevicesPath: pciDevicesPath, - procPath: filepath.Join(root, "proc"), - vfioDevicesPath: filepath.Join(root, "dev", "vfio", "devices"), - owners: make(map[string]string), - } - t.Cleanup(func() { hostVendorVFIO = originalVendorVFIO }) - require.NoError(t, os.MkdirAll(hostVendorVFIO.procPath, 0o755)) +func TestVGPUCleanupPendingErrorUnwraps(t *testing.T) { + t.Parallel() + + cause := errors.New("boot failed") + err := &VGPUCleanupPendingError{InstanceID: "inst-1", Err: cause} + assert.ErrorIs(t, err, cause) + assert.Contains(t, err.Error(), "inst-1") +} +func newStartRollbackVGPUManager(t *testing.T, destroy func(context.Context, devices.VGPUAssignment) error) (*manager, string) { + t.Helper() m := &manager{ paths: paths.New(t.TempDir()), imageManager: readyFixtureImageManager{name: "test-image"}, instanceLocks: sync.Map{}, bootMarkerScans: sync.Map{}, + createVGPU: func(_ context.Context, profileName, _ string) (*devices.VGPUDevice, error) { + return &devices.VGPUDevice{ + Framework: devices.VGPUFrameworkVendorVFIO, + VFAddress: "0000:82:00.4", + ProfileType: "1148", + ProfileName: profileName, + SysfsPath: "/sys/bus/pci/devices/0000:82:00.4", + }, nil + }, + destroyVGPU: destroy, } const id = "start-rollback" require.NoError(t, m.ensureDirectories(id)) @@ -102,25 +105,48 @@ func TestStartRollbackClearsVGPUAssignmentAfterSuccessfulDestroy(t *testing.T) { SocketPath: m.paths.InstanceSocket(id, "noop.sock"), DataDir: m.paths.InstanceDir(id), }})) + return m, id +} + +func TestStartRollbackClearsVGPUAssignmentAfterSuccessfulDestroy(t *testing.T) { + var destroyed []devices.VGPUAssignment + m, id := newStartRollbackVGPUManager(t, func(_ context.Context, assignment devices.VGPUAssignment) error { + destroyed = append(destroyed, assignment) + return nil + }) - t.Setenv("TMPDIR", filepath.Join(root, "missing")) + t.Setenv("TMPDIR", filepath.Join(t.TempDir(), "missing")) _, err := m.startInstance(context.Background(), id, StartInstanceRequest{}) require.Error(t, err) + require.Len(t, destroyed, 1) + assert.Equal(t, devices.VGPUAssignment{ + Framework: devices.VGPUFrameworkVendorVFIO, + DevicePath: "/sys/bus/pci/devices/0000:82:00.4", + InstanceID: id, + }, destroyed[0]) + stored, err := m.loadMetadata(id) require.NoError(t, err) assert.Equal(t, "NVIDIA L40S-2Q", stored.GPUProfile) assert.Empty(t, stored.GPUFramework) assert.Empty(t, stored.GPUDevicePath) assert.Empty(t, stored.GPUMdevUUID) - assertFileContents(t, filepath.Join(nvidiaPath, "current_vgpu_type"), "0") } -func assertFileContents(t *testing.T, path, want string) { - t.Helper() - got, err := os.ReadFile(path) +func TestStartRollbackRetainsVGPUAssignmentAfterFailedDestroy(t *testing.T) { + m, id := newStartRollbackVGPUManager(t, func(context.Context, devices.VGPUAssignment) error { + return errors.New("destroy failed") + }) + + t.Setenv("TMPDIR", filepath.Join(t.TempDir(), "missing")) + _, err := m.startInstance(context.Background(), id, StartInstanceRequest{}) + require.Error(t, err) + + stored, err := m.loadMetadata(id) require.NoError(t, err) - assert.Equal(t, want, string(got)) + assert.Equal(t, devices.VGPUFrameworkVendorVFIO, stored.GPUFramework) + assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", stored.GPUDevicePath) } func TestVGPUAssignmentClaimedByLiveInstanceFailsOnInvalidMetadata(t *testing.T) { From 38f14b1ef03b5094d229d191e9674f0d20d91ac8 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Fri, 7 Aug 2026 20:45:49 +0000 Subject: [PATCH 10/76] Generalize the create vGPU error text --- lib/instances/create.go | 2 +- lib/instances/create_mdev_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/instances/create.go b/lib/instances/create.go index dc70639fd..2cf34b5d9 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -56,7 +56,7 @@ func wrapCreateVGPUErr(profile string, err error) error { if errors.Is(err, devices.ErrVGPUNotSupportedOnMacOS) { return fmt.Errorf("%w: %w", ErrInvalidRequest, err) } - return fmt.Errorf("create vGPU mdev for profile %s: %w", profile, err) + return fmt.Errorf("create vGPU for profile %s: %w", profile, err) } // generateVsockCID converts first 8 chars of instance ID to a unique CID diff --git a/lib/instances/create_mdev_test.go b/lib/instances/create_mdev_test.go index e6e4f55c5..05b3e9d8c 100644 --- a/lib/instances/create_mdev_test.go +++ b/lib/instances/create_mdev_test.go @@ -63,7 +63,7 @@ func TestWrapCreateVGPUErr(t *testing.T) { { name: "other vGPU error", err: errors.New("boom"), - wantMessage: "create vGPU mdev for profile profile: boom", + wantMessage: "create vGPU for profile profile: boom", }, } { t.Run(tc.name, func(t *testing.T) { From 1411cbd43ef984833acc9233138b25b8a188701e Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Sat, 8 Aug 2026 00:58:24 +0000 Subject: [PATCH 11/76] Scope vGPU claim scan to vendor VFIO and close reconcile gaps --- cmd/api/api/instances.go | 14 +++++++------ cmd/api/api/instances_test.go | 31 ++++++++++++++++++++++++++++ cmd/api/main.go | 8 +++++-- cmd/api/main_test.go | 30 +++++++++++++++++++++++++++ lib/instances/lifecycle_noop_test.go | 4 ++-- lib/instances/vgpu.go | 15 +++++++++++--- lib/instances/vgpu_test.go | 21 +++++++++++++++++++ 7 files changed, 110 insertions(+), 13 deletions(-) diff --git a/cmd/api/api/instances.go b/cmd/api/api/instances.go index e9cee6ed9..2e2ef615a 100644 --- a/cmd/api/api/instances.go +++ b/cmd/api/api/instances.go @@ -364,6 +364,14 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst if err != nil { var vgpuPending *instances.VGPUCleanupPendingError switch { + // Checked first: it wraps the original create error, so a later + // errors.Is case would match the cause and hide the retained instance. + case errors.As(err, &vgpuPending): + log.ErrorContext(ctx, "failed to create instance", "error", err, "image", request.Body.Image) + return oapi.CreateInstance500JSONResponse{ + Code: "vgpu_cleanup_pending", + Message: fmt.Sprintf("failed to create instance; vGPU release failed during rollback and instance %s retains the assignment, delete it to retry", vgpuPending.InstanceID), + }, nil case errors.Is(err, instances.ErrImageNotReady): return oapi.CreateInstance400JSONResponse{ Code: "image_not_ready", @@ -414,12 +422,6 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst Code: "not_found", Message: err.Error(), }, nil - case errors.As(err, &vgpuPending): - log.ErrorContext(ctx, "failed to create instance", "error", err, "image", request.Body.Image) - return oapi.CreateInstance500JSONResponse{ - Code: "vgpu_cleanup_pending", - Message: fmt.Sprintf("failed to create instance; vGPU release failed during rollback and instance %s retains the assignment, delete it to retry", vgpuPending.InstanceID), - }, nil default: log.ErrorContext(ctx, "failed to create instance", "error", err, "image", request.Body.Image) return oapi.CreateInstance500JSONResponse{ diff --git a/cmd/api/api/instances_test.go b/cmd/api/api/instances_test.go index cc94c572d..b2bc46871 100644 --- a/cmd/api/api/instances_test.go +++ b/cmd/api/api/instances_test.go @@ -17,6 +17,7 @@ import ( "github.com/kernel/hypeman/lib/instances" "github.com/kernel/hypeman/lib/instances/phasetracking" mw "github.com/kernel/hypeman/lib/middleware" + "github.com/kernel/hypeman/lib/network" "github.com/kernel/hypeman/lib/oapi" "github.com/kernel/hypeman/lib/paths" restartpolicy "github.com/kernel/hypeman/lib/restart-policy" @@ -47,6 +48,36 @@ func TestGetInstance_NotFound(t *testing.T) { require.Error(t, err) } +type createErrorInstanceManager struct { + instances.Manager + err error +} + +func (m createErrorInstanceManager) CreateInstance(context.Context, instances.CreateInstanceRequest) (*instances.Instance, error) { + return nil, m.err +} + +// A retained-assignment error must win over the mapping of the create error +// it wraps, or the response omits the instance the caller has to delete. +func TestCreateInstance_VGPUCleanupPendingBeatsWrappedErrorMapping(t *testing.T) { + t.Parallel() + svc := newTestService(t) + svc.InstanceManager = createErrorInstanceManager{err: &instances.VGPUCleanupPendingError{ + InstanceID: "inst-1", + Err: network.ErrNameExists, + }} + + resp, err := svc.CreateInstance(ctx(), oapi.CreateInstanceRequestObject{ + Body: &oapi.CreateInstanceRequest{Image: "test-image"}, + }) + require.NoError(t, err) + + pending, ok := resp.(oapi.CreateInstance500JSONResponse) + require.True(t, ok, "expected 500 vgpu_cleanup_pending, got %T", resp) + assert.EqualValues(t, "vgpu_cleanup_pending", pending.Code) + assert.Contains(t, pending.Message, "inst-1") +} + func TestCreateInstance_AutoPullImage(t *testing.T) { t.Parallel() if _, err := os.Stat("/dev/kvm"); os.IsNotExist(err) { diff --git a/cmd/api/main.go b/cmd/api/main.go index 0479f1583..3374d9682 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -192,10 +192,14 @@ func liveInstanceVGPUDevicePaths(ctx context.Context, instanceManager instances. } protected := make(map[string]struct{}) for _, inst := range allInstances { - if inst.GPUDevicePath == "" || inst.HypervisorPID == nil { + if inst.GPUDevicePath == "" { continue } - if !instances.HypervisorProcessExists(*inst.HypervisorPID, inst.SocketPath) { + // A nil PID does not mean the assignment is orphaned: the PID is + // persisted only after the hypervisor starts, so a crash during boot + // leaves the device path without one. Only skip protection when the + // recorded hypervisor is known to be gone. + if inst.HypervisorPID != nil && !instances.HypervisorProcessExists(*inst.HypervisorPID, inst.SocketPath) { continue } protected[inst.GPUDevicePath] = struct{}{} diff --git a/cmd/api/main_test.go b/cmd/api/main_test.go index 34dbba428..573a404c1 100644 --- a/cmd/api/main_test.go +++ b/cmd/api/main_test.go @@ -2,15 +2,18 @@ package main import ( "bytes" + "context" "net/http" "net/http/httptest" "net/url" + "os/exec" "testing" "time" "github.com/getkin/kin-openapi/openapi3filter" "github.com/go-chi/chi/v5" "github.com/golang-jwt/jwt/v5" + "github.com/kernel/hypeman/lib/instances" mw "github.com/kernel/hypeman/lib/middleware" "github.com/kernel/hypeman/lib/oapi" nethttpmiddleware "github.com/oapi-codegen/nethttp-middleware" @@ -338,3 +341,30 @@ func TestImageNameWithSlashes_URLEncoding(t *testing.T) { }) } } + +type vgpuReconcileManagerStub struct { + instances.Manager + list []instances.Instance +} + +func (s vgpuReconcileManagerStub) ListInstancesForReconcile(context.Context) ([]instances.Instance, error) { + return s.list, nil +} + +// The hypervisor PID is persisted only after boot, so an assignment without +// one may belong to a VM that is still starting and must stay protected. +func TestLiveInstanceVGPUDevicePathsProtectsAssignmentsWithoutPID(t *testing.T) { + dead := exec.Command("true") + require.NoError(t, dead.Run()) + deadPID := dead.Process.Pid + + manager := vgpuReconcileManagerStub{list: []instances.Instance{ + {StoredMetadata: instances.StoredMetadata{Id: "booting", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4"}}, + {StoredMetadata: instances.StoredMetadata{Id: "dead", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.5", HypervisorPID: &deadPID}}, + }} + + protected, err := liveInstanceVGPUDevicePaths(context.Background(), manager) + require.NoError(t, err) + assert.Contains(t, protected, "/sys/bus/pci/devices/0000:82:00.4") + assert.NotContains(t, protected, "/sys/bus/pci/devices/0000:82:00.5") +} diff --git a/lib/instances/lifecycle_noop_test.go b/lib/instances/lifecycle_noop_test.go index f3baa9dae..15f632b3a 100644 --- a/lib/instances/lifecycle_noop_test.go +++ b/lib/instances/lifecycle_noop_test.go @@ -202,7 +202,7 @@ func TestDeleteDropsStaleVGPUClaimedByLiveInstance(t *testing.T) { meta, err := m.loadMetadata(id) require.NoError(t, err) meta.GPUProfile = "NVIDIA L40S-2Q" - meta.GPUFramework = devices.VGPUFramework("future-framework") + meta.GPUFramework = devices.VGPUFrameworkVendorVFIO meta.GPUDevicePath = "/sys/bus/pci/devices/0000:82:00.4" require.NoError(t, m.saveMetadata(meta)) @@ -230,7 +230,7 @@ func TestDeleteDropsStaleVGPUClaimedByLiveInstance(t *testing.T) { SocketPath: socketPath, DataDir: m.paths.InstanceDir(claimantID), GPUProfile: "NVIDIA L40S-2Q", - GPUFramework: devices.VGPUFramework("future-framework"), + GPUFramework: devices.VGPUFrameworkVendorVFIO, GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", }})) diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index ae100a39e..9b97fddb7 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -61,9 +61,18 @@ func clearStoredVGPUDevice(stored *StoredMetadata) { func (m *manager) releaseStoredVGPU(ctx context.Context, stored *StoredMetadata) error { path := storedVGPUDevicePath(stored) if path != "" { - claimed, err := m.vgpuAssignmentClaimedByLiveInstance(ctx, stored.Id, path) - if err != nil { - return err + // Vendor VFIO VFs are reused across instances, so stale metadata can + // point at a path claimed by a live instance and the release must fail + // closed on an incomplete inventory. mdev UUIDs are unique and never + // reused, so skip the scan there — it would let one unreadable + // metadata file block every mdev release on the host. + claimed := false + if stored.GPUFramework == devices.VGPUFrameworkVendorVFIO { + var err error + claimed, err = m.vgpuAssignmentClaimedByLiveInstance(ctx, stored.Id, path) + if err != nil { + return err + } } if claimed { logger.FromContext(ctx).WarnContext(ctx, "dropping stale vGPU assignment claimed by another live instance", diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index da8426efe..6f9a20073 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -178,6 +178,27 @@ func TestVGPUAssignmentClaimedByLiveInstanceNormalizesLegacyMdevPath(t *testing. assert.True(t, claimed) } +func TestReleaseStoredVGPUSkipsClaimScanForMdev(t *testing.T) { + t.Parallel() + + m := &manager{ + paths: paths.New(t.TempDir()), + destroyVGPU: func(context.Context, devices.VGPUAssignment) error { return nil }, + } + require.NoError(t, m.ensureDirectories("invalid-instance")) + require.NoError(t, os.WriteFile(m.paths.InstanceMetadata("invalid-instance"), []byte("{"), 0o644)) + + stored := &StoredMetadata{ + Id: "mdev-instance", + GPUFramework: devices.VGPUFrameworkMdev, + GPUMdevUUID: "uuid-1", + GPUDevicePath: "/sys/bus/mdev/devices/uuid-1", + } + require.NoError(t, m.releaseStoredVGPU(context.Background(), stored), + "an unreadable metadata file must not block mdev releases") + assert.Empty(t, stored.GPUDevicePath) +} + func TestStoredVGPUDevicePath(t *testing.T) { t.Parallel() From 520533a92dd5f28afc9500a2f0050fccd9eb88fe Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Sun, 9 Aug 2026 06:25:10 +0000 Subject: [PATCH 12/76] Harden the vendor VFIO release path Enable vendor VFIO dispatch in CreateVGPU now that the lifecycle persists assignments durably and guards releases. Protect nil-PID claims in the release guard: the hypervisor PID is only persisted after the claimant boots, so a matching assignment without a PID must be treated as live, matching the startup reconcile protection. Scan raw metadata instead of hydrating instances for the claim check. Hydration derives state through hypervisor queries for every instance on the host, which every vendor VFIO release would pay; the guard only needs the stored assignment, PID, and socket. Unreadable metadata still fails the release closed. Report pending vGPU cleanup even when retaining the rollback record fails: the destroy already failed, so the caller must learn about the outstanding assignment either way. --- integration/vgpu_test.go | 5 ---- lib/devices/vgpu_linux.go | 5 +--- lib/instances/create.go | 11 +++++--- lib/instances/vgpu.go | 35 ++++++++++++++++++++++---- lib/instances/vgpu_test.go | 51 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 89 insertions(+), 18 deletions(-) diff --git a/integration/vgpu_test.go b/integration/vgpu_test.go index 7873aa35c..1f1a82776 100644 --- a/integration/vgpu_test.go +++ b/integration/vgpu_test.go @@ -324,11 +324,6 @@ func checkVGPUTestPrerequisites() (string, string) { if framework == devices.VGPUFrameworkNone { return "vGPU test requires SR-IOV VFs with an mdev or vendor VFIO vGPU framework", "" } - if framework == devices.VGPUFrameworkVendorVFIO { - // CreateVGPU rejects vendor VFIO until the instance lifecycle - // integration lands. - return "vGPU test requires the vendor VFIO instance lifecycle integration", "" - } // Check for available profiles profiles, err := devices.ListGPUProfiles() diff --git a/lib/devices/vgpu_linux.go b/lib/devices/vgpu_linux.go index b3c497899..be92e8ee6 100644 --- a/lib/devices/vgpu_linux.go +++ b/lib/devices/vgpu_linux.go @@ -73,10 +73,7 @@ func CreateVGPU(ctx context.Context, profileName, instanceID string) (*VGPUDevic MdevUUID: mdev.UUID, }, nil case VGPUFrameworkVendorVFIO: - // The instance lifecycle does not yet persist vendor VFIO assignments - // durably or guard their release against live claims, so keep the - // backend out of the create path until that integration lands. - return nil, fmt.Errorf("vendor VFIO vGPU support is not yet integrated with the instance lifecycle") + return hostVendorVFIO.create(ctx, profileName, instanceID) default: return nil, fmt.Errorf("vGPU framework not available") } diff --git a/lib/instances/create.go b/lib/instances/create.go index 2cf34b5d9..f1edb8aaa 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -641,8 +641,11 @@ func resolveCreateExpiration(req CreateInstanceRequest, now time.Time) (*time.Ti return &expiresAt, nil } -// cleanupFailedCreate reports whether it retained instance metadata for a -// vGPU assignment whose release failed during rollback. +// cleanupFailedCreate reports whether a vGPU assignment is still outstanding +// after a failed create. The vGPU destroy already failed when retainedVGPU is +// set, so the pending cleanup is reported even when the retention record +// cannot be persisted — in that case the assignment is orphaned until the +// next startup reconcile, and the caller must still surface it. func (m *manager) cleanupFailedCreate(ctx context.Context, id string, retainedVGPU *StoredMetadata) bool { if retainedVGPU == nil { m.deleteInstanceData(id) @@ -652,7 +655,7 @@ func (m *manager) cleanupFailedCreate(ctx context.Context, id string, retainedVG log := logger.FromContext(ctx) if err := m.ensureDirectories(id); err != nil { log.ErrorContext(ctx, "failed to retain instance data after vGPU cleanup failure", "instance_id", id, "error", err) - return false + return true } retained := StoredMetadata{ Id: id, @@ -662,7 +665,7 @@ func (m *manager) cleanupFailedCreate(ctx context.Context, id string, retainedVG } if err := m.saveMetadata(&metadata{StoredMetadata: retained}); err != nil { log.ErrorContext(ctx, "failed to retain vGPU assignment metadata after cleanup failure", "instance_id", id, "error", err) - return false + return true } return true } diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index 9b97fddb7..d62fc0d52 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -93,19 +93,44 @@ func (m *manager) releaseStoredVGPU(ctx context.Context, stored *StoredMetadata) return nil } +// vgpuAssignmentClaimedByLiveInstance reports whether another instance's +// stored metadata claims devicePath. It reads raw metadata instead of +// hydrating full instances: the scan runs on every vendor VFIO release, and +// deriving state would query the hypervisor of every instance on the host. +// It fails closed: unreadable metadata is an error, and a matching claim +// without a persisted PID counts as live because the PID is only persisted +// after the claimant's hypervisor starts. func (m *manager) vgpuAssignmentClaimedByLiveInstance(ctx context.Context, excludeID, devicePath string) (bool, error) { - instances, err := m.ListInstancesForReconcile(ctx) + files, err := m.listMetadataFilesWithStatErrors(true) if err != nil { return false, fmt.Errorf("list instances for vGPU release check: %w", err) } - for i := range instances { - inst := &instances[i] - if inst.Id == excludeID || storedVGPUDevicePath(&inst.StoredMetadata) != devicePath || inst.HypervisorPID == nil { + for _, file := range files { + id := filepath.Base(filepath.Dir(file)) + if id == excludeID { continue } - if HypervisorProcessExists(*inst.HypervisorPID, inst.SocketPath) { + meta, err := m.loadMetadata(id) + if err != nil { + return false, fmt.Errorf("load metadata for vGPU release check: instance %s: %w", id, err) + } + stored := &meta.StoredMetadata + if storedVGPUDevicePath(stored) != devicePath { + continue + } + if stored.HypervisorPID == nil { return true, nil } + if HypervisorProcessExists(*stored.HypervisorPID, stored.SocketPath) { + return true, nil + } + // The stored PID can be stale after a hypeman restart; a live owner + // of the claimant's socket still marks the claim as live. + if stored.SocketPath != "" && !ProcessExists(*stored.HypervisorPID) { + if owner, _, err := hypervisor.ResolveProcessPID(stored.SocketPath); err == nil && ProcessExists(owner) { + return true, nil + } + } } return false, nil } diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index 6f9a20073..0253d83da 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -67,6 +67,23 @@ func TestCleanupFailedCreateDeletesDataWithoutRetainedVGPU(t *testing.T) { require.Error(t, err) } +func TestCleanupFailedCreateReportsPendingWhenRetentionFails(t *testing.T) { + t.Parallel() + + m := &manager{paths: paths.New(t.TempDir())} + // A file at the guests directory path makes ensureDirectories fail even + // when running as root. + require.NoError(t, os.WriteFile(m.paths.GuestsDir(), nil, 0o644)) + + stored := &StoredMetadata{ + Id: "failed-create", + GPUFramework: devices.VGPUFrameworkVendorVFIO, + GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", + } + assert.True(t, m.cleanupFailedCreate(context.Background(), stored.Id, stored), + "a failed retention must still report the outstanding vGPU assignment") +} + func TestVGPUCleanupPendingErrorUnwraps(t *testing.T) { t.Parallel() @@ -178,6 +195,40 @@ func TestVGPUAssignmentClaimedByLiveInstanceNormalizesLegacyMdevPath(t *testing. assert.True(t, claimed) } +func TestVGPUAssignmentClaimedByLiveInstanceProtectsNilPIDClaim(t *testing.T) { + t.Parallel() + + m := &manager{paths: paths.New(t.TempDir())} + require.NoError(t, m.ensureDirectories("booting-claimant")) + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: "booting-claimant", + Name: "booting-claimant", + GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", + }})) + + claimed, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "other-instance", "/sys/bus/pci/devices/0000:82:00.4") + require.NoError(t, err) + assert.True(t, claimed, "a matching claim without a persisted PID must be treated as live: the PID is only persisted after the claimant boots") +} + +func TestVGPUAssignmentClaimedByLiveInstanceIgnoresDeadClaim(t *testing.T) { + t.Parallel() + + m := &manager{paths: paths.New(t.TempDir())} + require.NoError(t, m.ensureDirectories("dead-claimant")) + deadPID := 1 << 30 + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: "dead-claimant", + Name: "dead-claimant", + GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", + HypervisorPID: &deadPID, + }})) + + claimed, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "other-instance", "/sys/bus/pci/devices/0000:82:00.4") + require.NoError(t, err) + assert.False(t, claimed, "a claim whose hypervisor is gone must not block the release") +} + func TestReleaseStoredVGPUSkipsClaimScanForMdev(t *testing.T) { t.Parallel() From 7e1737c3bb78ab73f71998307b5fb639670d659b Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Sun, 9 Aug 2026 07:40:01 +0000 Subject: [PATCH 13/76] Fail closed on retained vGPU cleanup --- lib/instances/create.go | 26 +++++------- lib/instances/process_identity_linux_test.go | 43 ++++++++++++++++++++ lib/instances/vgpu.go | 21 ++++------ lib/instances/vgpu_test.go | 23 ++++++----- 4 files changed, 76 insertions(+), 37 deletions(-) diff --git a/lib/instances/create.go b/lib/instances/create.go index f1edb8aaa..de00bdcf3 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -281,19 +281,18 @@ func (m *manager) createInstance( var retainedVGPU *StoredMetadata // Setup cleanup stack early so device attachment errors trigger cleanup. - // When rollback retains a vGPU assignment, surface the retained instance - // ID to the caller so the record is discoverable and can be deleted to - // retry the release. The wrapping defer is registered first so it runs - // after cu.Clean has decided whether metadata was retained. - vgpuRetained := false + // When rollback cannot release a vGPU assignment, report whether its + // retention record was persisted. The wrapping defer is registered first + // so it runs after cu.Clean has attempted to retain the metadata. + vgpuPersisted := false defer func() { - if retErr != nil && vgpuRetained { - retErr = &VGPUCleanupPendingError{InstanceID: id, Err: retErr} + if retErr != nil && retainedVGPU != nil { + retErr = &VGPUCleanupPendingError{InstanceID: id, Retained: vgpuPersisted, Err: retErr} } }() cu := cleanup.Make(func() { log.DebugContext(ctx, "cleaning up instance on error", "instance_id", id) - vgpuRetained = m.cleanupFailedCreate(ctx, id, retainedVGPU) + vgpuPersisted = m.cleanupFailedCreate(ctx, id, retainedVGPU) }) defer cu.Clean() @@ -641,11 +640,8 @@ func resolveCreateExpiration(req CreateInstanceRequest, now time.Time) (*time.Ti return &expiresAt, nil } -// cleanupFailedCreate reports whether a vGPU assignment is still outstanding -// after a failed create. The vGPU destroy already failed when retainedVGPU is -// set, so the pending cleanup is reported even when the retention record -// cannot be persisted — in that case the assignment is orphaned until the -// next startup reconcile, and the caller must still surface it. +// cleanupFailedCreate reports whether the retention record for a vGPU +// assignment whose release failed during rollback was persisted. func (m *manager) cleanupFailedCreate(ctx context.Context, id string, retainedVGPU *StoredMetadata) bool { if retainedVGPU == nil { m.deleteInstanceData(id) @@ -655,7 +651,7 @@ func (m *manager) cleanupFailedCreate(ctx context.Context, id string, retainedVG log := logger.FromContext(ctx) if err := m.ensureDirectories(id); err != nil { log.ErrorContext(ctx, "failed to retain instance data after vGPU cleanup failure", "instance_id", id, "error", err) - return true + return false } retained := StoredMetadata{ Id: id, @@ -665,7 +661,7 @@ func (m *manager) cleanupFailedCreate(ctx context.Context, id string, retainedVG } if err := m.saveMetadata(&metadata{StoredMetadata: retained}); err != nil { log.ErrorContext(ctx, "failed to retain vGPU assignment metadata after cleanup failure", "instance_id", id, "error", err) - return true + return false } return true } diff --git a/lib/instances/process_identity_linux_test.go b/lib/instances/process_identity_linux_test.go index 7450ed112..d5bd973d9 100644 --- a/lib/instances/process_identity_linux_test.go +++ b/lib/instances/process_identity_linux_test.go @@ -17,7 +17,9 @@ import ( "testing" "time" + "github.com/kernel/hypeman/lib/devices" "github.com/kernel/hypeman/lib/hypervisor" + "github.com/kernel/hypeman/lib/paths" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -508,6 +510,47 @@ func TestRefreshHypervisorPIDResolvesSocketOwnerWhenStoredPIDIsDead(t *testing.T assert.Equal(t, hostBootID(), stored.HypervisorBootID) } +func TestVGPUAssignmentClaimedByLiveInstanceProtectsReusedPIDClaim(t *testing.T) { + socketPath := filepath.Join(t.TempDir(), "test.sock") + owner := exec.Command(os.Args[0], "-test.run=^TestHypervisorProcessExistsWithReboundSocketPathHelper$") + owner.Env = append(os.Environ(), "HYPERVISOR_SOCKET_HELPER=1", "HYPERVISOR_SOCKET_PATH="+socketPath) + stdin, err := owner.StdinPipe() + require.NoError(t, err) + stdout, err := owner.StdoutPipe() + require.NoError(t, err) + require.NoError(t, owner.Start()) + t.Cleanup(func() { + _ = stdin.Close() + _ = owner.Process.Kill() + _ = owner.Wait() + }) + _, err = bufio.NewReader(stdout).ReadString('\n') + require.NoError(t, err) + + stale := exec.Command("sleep", "30") + require.NoError(t, stale.Start()) + t.Cleanup(func() { + _ = stale.Process.Kill() + _ = stale.Wait() + }) + + m := &manager{paths: paths.New(t.TempDir())} + const devicePath = "/sys/bus/pci/devices/0000:82:00.4" + stalePID := stale.Process.Pid + require.NoError(t, m.ensureDirectories("live-claimant")) + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: "live-claimant", + GPUFramework: devices.VGPUFrameworkVendorVFIO, + GPUDevicePath: devicePath, + HypervisorPID: &stalePID, + SocketPath: socketPath, + }})) + + claimed, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "other-instance", devicePath) + require.NoError(t, err) + assert.True(t, claimed) +} + func TestKillHypervisorSurvivesConcurrentReaper(t *testing.T) { socketPath := filepath.Join(t.TempDir(), "test.sock") process := exec.Command(os.Args[0], "-test.run=^TestSocketListenerHelper$") diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index d62fc0d52..e27cf7a54 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -5,7 +5,6 @@ import ( "path/filepath" "github.com/kernel/hypeman/lib/devices" - "github.com/kernel/hypeman/lib/hypervisor" "github.com/kernel/hypeman/lib/logger" ) @@ -17,15 +16,19 @@ func validateVGPUHypervisor(hvType hypervisor.Type) error { } // VGPUCleanupPendingError reports a failed create whose vGPU release also -// failed during rollback. The instance record identified by InstanceID is -// retained so the release can be retried; deleting the instance retries it. +// failed during rollback. When Retained is true, deleting the retained instance +// retries the release; otherwise startup reconciliation recovers the assignment. type VGPUCleanupPendingError struct { InstanceID string + Retained bool Err error } func (e *VGPUCleanupPendingError) Error() string { - return fmt.Sprintf("%v; vGPU release failed during rollback, instance %s retains the assignment", e.Err, e.InstanceID) + if e.Retained { + return fmt.Sprintf("%v; vGPU release failed during rollback, instance %s retains the assignment", e.Err, e.InstanceID) + } + return fmt.Sprintf("%v; vGPU release failed during rollback and the retention record for instance %s could not be saved; the assignment is recovered on the next startup reconcile", e.Err, e.InstanceID) } func (e *VGPUCleanupPendingError) Unwrap() error { return e.Err } @@ -121,16 +124,10 @@ func (m *manager) vgpuAssignmentClaimedByLiveInstance(ctx context.Context, exclu if stored.HypervisorPID == nil { return true, nil } - if HypervisorProcessExists(*stored.HypervisorPID, stored.SocketPath) { + pid, err := resolveLiveHypervisorPID(stored.HypervisorPID, stored.SocketPath) + if err != nil || pid > 0 { return true, nil } - // The stored PID can be stale after a hypeman restart; a live owner - // of the claimant's socket still marks the claim as live. - if stored.SocketPath != "" && !ProcessExists(*stored.HypervisorPID) { - if owner, _, err := hypervisor.ResolveProcessPID(stored.SocketPath); err == nil && ProcessExists(owner) { - return true, nil - } - } } return false, nil } diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index 0253d83da..a0d3ae9da 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -67,30 +67,33 @@ func TestCleanupFailedCreateDeletesDataWithoutRetainedVGPU(t *testing.T) { require.Error(t, err) } -func TestCleanupFailedCreateReportsPendingWhenRetentionFails(t *testing.T) { +func TestCleanupFailedCreateReportsUnpersistedRetention(t *testing.T) { t.Parallel() m := &manager{paths: paths.New(t.TempDir())} - // A file at the guests directory path makes ensureDirectories fail even - // when running as root. - require.NoError(t, os.WriteFile(m.paths.GuestsDir(), nil, 0o644)) + const id = "failed-create" + require.NoError(t, m.ensureDirectories(id)) + require.NoError(t, os.Mkdir(m.paths.InstanceMetadata(id), 0o755)) stored := &StoredMetadata{ - Id: "failed-create", + Id: id, GPUFramework: devices.VGPUFrameworkVendorVFIO, GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", } - assert.True(t, m.cleanupFailedCreate(context.Background(), stored.Id, stored), - "a failed retention must still report the outstanding vGPU assignment") + assert.False(t, m.cleanupFailedCreate(context.Background(), stored.Id, stored)) } func TestVGPUCleanupPendingErrorUnwraps(t *testing.T) { t.Parallel() cause := errors.New("boot failed") - err := &VGPUCleanupPendingError{InstanceID: "inst-1", Err: cause} - assert.ErrorIs(t, err, cause) - assert.Contains(t, err.Error(), "inst-1") + retained := &VGPUCleanupPendingError{InstanceID: "inst-1", Retained: true, Err: cause} + assert.ErrorIs(t, retained, cause) + assert.Equal(t, "boot failed; vGPU release failed during rollback, instance inst-1 retains the assignment", retained.Error()) + + unpersisted := &VGPUCleanupPendingError{InstanceID: "inst-1", Err: cause} + assert.ErrorIs(t, unpersisted, cause) + assert.Equal(t, "boot failed; vGPU release failed during rollback and the retention record for instance inst-1 could not be saved; the assignment is recovered on the next startup reconcile", unpersisted.Error()) } func newStartRollbackVGPUManager(t *testing.T, destroy func(context.Context, devices.VGPUAssignment) error) (*manager, string) { From 928f8d2226f504633d377ca578ddd37613d4dbe0 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Sun, 9 Aug 2026 22:02:02 +0000 Subject: [PATCH 14/76] Report surviving vGPU retention metadata --- lib/instances/create.go | 14 ++++++++++++-- lib/instances/vgpu_test.go | 28 ++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/lib/instances/create.go b/lib/instances/create.go index de00bdcf3..8af298b66 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -649,9 +649,19 @@ func (m *manager) cleanupFailedCreate(ctx context.Context, id string, retainedVG } log := logger.FromContext(ctx) + retentionFailed := func() bool { + meta, err := m.loadMetadata(id) + if err == nil && storedVGPUDevicePath(&meta.StoredMetadata) != "" { + return true + } + if err := m.deleteInstanceData(id); err != nil { + log.ErrorContext(ctx, "failed to delete stale instance data after retention failure", "instance_id", id, "error", err) + } + return false + } if err := m.ensureDirectories(id); err != nil { log.ErrorContext(ctx, "failed to retain instance data after vGPU cleanup failure", "instance_id", id, "error", err) - return false + return retentionFailed() } retained := StoredMetadata{ Id: id, @@ -661,7 +671,7 @@ func (m *manager) cleanupFailedCreate(ctx context.Context, id string, retainedVG } if err := m.saveMetadata(&metadata{StoredMetadata: retained}); err != nil { log.ErrorContext(ctx, "failed to retain vGPU assignment metadata after cleanup failure", "instance_id", id, "error", err) - return false + return retentionFailed() } return true } diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index a0d3ae9da..b6831109c 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -81,6 +81,34 @@ func TestCleanupFailedCreateReportsUnpersistedRetention(t *testing.T) { GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", } assert.False(t, m.cleanupFailedCreate(context.Background(), stored.Id, stored)) + _, err := m.loadMetadata(id) + require.Error(t, err) +} + +func TestCleanupFailedCreateReportsRetainedWhenFullMetadataSurvives(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("root bypasses directory permissions") + } + + m := &manager{paths: paths.New(t.TempDir())} + const id = "failed-create" + require.NoError(t, m.ensureDirectories(id)) + stored := &StoredMetadata{ + Id: id, + GPUFramework: devices.VGPUFrameworkVendorVFIO, + GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", + } + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: *stored})) + + instanceDir := filepath.Dir(m.paths.InstanceMetadata(id)) + require.NoError(t, os.Chmod(instanceDir, 0o555)) + t.Cleanup(func() { _ = os.Chmod(instanceDir, 0o755) }) + + assert.True(t, m.cleanupFailedCreate(context.Background(), id, stored)) + retained, err := m.loadMetadata(id) + require.NoError(t, err) + assert.Equal(t, stored.GPUFramework, retained.GPUFramework) + assert.Equal(t, stored.GPUDevicePath, retained.GPUDevicePath) } func TestVGPUCleanupPendingErrorUnwraps(t *testing.T) { From 3f80948246a1ed30f2e5c24d63a06b0d6adeaa0b Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Sun, 9 Aug 2026 22:02:38 +0000 Subject: [PATCH 15/76] Return accurate vGPU cleanup guidance --- cmd/api/api/instances.go | 8 ++++++-- cmd/api/api/instances_test.go | 23 +++++++++++++++++++++++ 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/cmd/api/api/instances.go b/cmd/api/api/instances.go index 2e2ef615a..d62f1d7d2 100644 --- a/cmd/api/api/instances.go +++ b/cmd/api/api/instances.go @@ -365,12 +365,16 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst var vgpuPending *instances.VGPUCleanupPendingError switch { // Checked first: it wraps the original create error, so a later - // errors.Is case would match the cause and hide the retained instance. + // errors.Is case would match the cause and hide the pending vGPU cleanup. case errors.As(err, &vgpuPending): log.ErrorContext(ctx, "failed to create instance", "error", err, "image", request.Body.Image) + message := fmt.Sprintf("failed to create instance; vGPU release failed during rollback and instance %s retains the assignment, delete it to retry", vgpuPending.InstanceID) + if !vgpuPending.Retained { + message = fmt.Sprintf("failed to create instance; vGPU release failed during rollback and the retention record for instance %s could not be saved; the assignment is recovered on the next startup reconcile", vgpuPending.InstanceID) + } return oapi.CreateInstance500JSONResponse{ Code: "vgpu_cleanup_pending", - Message: fmt.Sprintf("failed to create instance; vGPU release failed during rollback and instance %s retains the assignment, delete it to retry", vgpuPending.InstanceID), + Message: message, }, nil case errors.Is(err, instances.ErrImageNotReady): return oapi.CreateInstance400JSONResponse{ diff --git a/cmd/api/api/instances_test.go b/cmd/api/api/instances_test.go index b2bc46871..eec8e6dbb 100644 --- a/cmd/api/api/instances_test.go +++ b/cmd/api/api/instances_test.go @@ -64,6 +64,7 @@ func TestCreateInstance_VGPUCleanupPendingBeatsWrappedErrorMapping(t *testing.T) svc := newTestService(t) svc.InstanceManager = createErrorInstanceManager{err: &instances.VGPUCleanupPendingError{ InstanceID: "inst-1", + Retained: true, Err: network.ErrNameExists, }} @@ -76,6 +77,28 @@ func TestCreateInstance_VGPUCleanupPendingBeatsWrappedErrorMapping(t *testing.T) require.True(t, ok, "expected 500 vgpu_cleanup_pending, got %T", resp) assert.EqualValues(t, "vgpu_cleanup_pending", pending.Code) assert.Contains(t, pending.Message, "inst-1") + assert.Contains(t, pending.Message, "delete it to retry") +} + +func TestCreateInstance_VGPUCleanupPendingWithoutRetentionUsesReconcileGuidance(t *testing.T) { + t.Parallel() + svc := newTestService(t) + svc.InstanceManager = createErrorInstanceManager{err: &instances.VGPUCleanupPendingError{ + InstanceID: "inst-1", + Err: network.ErrNameExists, + }} + + resp, err := svc.CreateInstance(ctx(), oapi.CreateInstanceRequestObject{ + Body: &oapi.CreateInstanceRequest{Image: "test-image"}, + }) + require.NoError(t, err) + + pending, ok := resp.(oapi.CreateInstance500JSONResponse) + require.True(t, ok, "expected 500 vgpu_cleanup_pending, got %T", resp) + assert.EqualValues(t, "vgpu_cleanup_pending", pending.Code) + assert.Contains(t, pending.Message, "retention record for instance inst-1 could not be saved") + assert.Contains(t, pending.Message, "startup reconcile") + assert.NotContains(t, pending.Message, "delete") } func TestCreateInstance_AutoPullImage(t *testing.T) { From 59601959569b1ea30913c86ca304d684b12bec99 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Sun, 9 Aug 2026 22:26:00 +0000 Subject: [PATCH 16/76] Clarify vGPU retention fallback --- lib/instances/create.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/instances/create.go b/lib/instances/create.go index 8af298b66..42fdcac39 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -649,7 +649,7 @@ func (m *manager) cleanupFailedCreate(ctx context.Context, id string, retainedVG } log := logger.FromContext(ctx) - retentionFailed := func() bool { + retentionSurvives := func() bool { meta, err := m.loadMetadata(id) if err == nil && storedVGPUDevicePath(&meta.StoredMetadata) != "" { return true @@ -661,7 +661,7 @@ func (m *manager) cleanupFailedCreate(ctx context.Context, id string, retainedVG } if err := m.ensureDirectories(id); err != nil { log.ErrorContext(ctx, "failed to retain instance data after vGPU cleanup failure", "instance_id", id, "error", err) - return retentionFailed() + return retentionSurvives() } retained := StoredMetadata{ Id: id, @@ -671,7 +671,7 @@ func (m *manager) cleanupFailedCreate(ctx context.Context, id string, retainedVG } if err := m.saveMetadata(&metadata{StoredMetadata: retained}); err != nil { log.ErrorContext(ctx, "failed to retain vGPU assignment metadata after cleanup failure", "instance_id", id, "error", err) - return retentionFailed() + return retentionSurvives() } return true } From f677f8bfcd265b634550b3fc7fed8c67ec004658 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Mon, 10 Aug 2026 13:43:17 +0000 Subject: [PATCH 17/76] Pass hypervisor identity token to vGPU claim check --- lib/instances/vgpu.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index e27cf7a54..acc5eb749 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -124,7 +124,7 @@ func (m *manager) vgpuAssignmentClaimedByLiveInstance(ctx context.Context, exclu if stored.HypervisorPID == nil { return true, nil } - pid, err := resolveLiveHypervisorPID(stored.HypervisorPID, stored.SocketPath) + pid, err := resolveLiveHypervisorPID(stored.HypervisorPID, stored.HypervisorStartTime, stored.SocketPath) if err != nil || pid > 0 { return true, nil } From 51508f0e3a336d5ba88e2e55414c44a3a6b022f2 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Mon, 10 Aug 2026 14:12:07 +0000 Subject: [PATCH 18/76] Fail safely on ambiguous vGPU claims --- lib/instances/vgpu.go | 15 +++++++++------ lib/instances/vgpu_test.go | 38 ++++++++++++++++++++++++++++++++++---- 2 files changed, 43 insertions(+), 10 deletions(-) diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index acc5eb749..9bca44dbc 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -96,13 +96,13 @@ func (m *manager) releaseStoredVGPU(ctx context.Context, stored *StoredMetadata) return nil } -// vgpuAssignmentClaimedByLiveInstance reports whether another instance's +// vgpuAssignmentClaimedByLiveInstance reports whether another live instance's // stored metadata claims devicePath. It reads raw metadata instead of // hydrating full instances: the scan runs on every vendor VFIO release, and // deriving state would query the hypervisor of every instance on the host. -// It fails closed: unreadable metadata is an error, and a matching claim -// without a persisted PID counts as live because the PID is only persisted -// after the claimant's hypervisor starts. +// A confirmed live claimant returns true. Unreadable metadata, a missing PID, +// or unverifiable process ownership returns an error so the requester retains +// its assignment for a later retry. func (m *manager) vgpuAssignmentClaimedByLiveInstance(ctx context.Context, excludeID, devicePath string) (bool, error) { files, err := m.listMetadataFilesWithStatErrors(true) if err != nil { @@ -122,10 +122,13 @@ func (m *manager) vgpuAssignmentClaimedByLiveInstance(ctx context.Context, exclu continue } if stored.HypervisorPID == nil { - return true, nil + return false, fmt.Errorf("cannot confirm liveness of vGPU claimant %s on %s: no persisted hypervisor PID", id, devicePath) } pid, err := resolveLiveHypervisorPID(stored.HypervisorPID, stored.HypervisorStartTime, stored.SocketPath) - if err != nil || pid > 0 { + if err != nil { + return false, fmt.Errorf("cannot confirm liveness of vGPU claimant %s on %s: %w", id, devicePath, err) + } + if pid > 0 { return true, nil } } diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index b6831109c..d4e0c6489 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -226,7 +226,7 @@ func TestVGPUAssignmentClaimedByLiveInstanceNormalizesLegacyMdevPath(t *testing. assert.True(t, claimed) } -func TestVGPUAssignmentClaimedByLiveInstanceProtectsNilPIDClaim(t *testing.T) { +func TestVGPUAssignmentClaimedByLiveInstanceErrorsOnNilPIDClaim(t *testing.T) { t.Parallel() m := &manager{paths: paths.New(t.TempDir())} @@ -237,9 +237,9 @@ func TestVGPUAssignmentClaimedByLiveInstanceProtectsNilPIDClaim(t *testing.T) { GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", }})) - claimed, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "other-instance", "/sys/bus/pci/devices/0000:82:00.4") - require.NoError(t, err) - assert.True(t, claimed, "a matching claim without a persisted PID must be treated as live: the PID is only persisted after the claimant boots") + _, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "other-instance", "/sys/bus/pci/devices/0000:82:00.4") + require.Error(t, err) + assert.Contains(t, err.Error(), "booting-claimant") } func TestVGPUAssignmentClaimedByLiveInstanceIgnoresDeadClaim(t *testing.T) { @@ -281,6 +281,36 @@ func TestReleaseStoredVGPUSkipsClaimScanForMdev(t *testing.T) { assert.Empty(t, stored.GPUDevicePath) } +func TestReleaseStoredVGPURetainsRequesterOnAmbiguousClaim(t *testing.T) { + t.Parallel() + + const devicePath = "/sys/bus/pci/devices/0000:82:00.4" + m := &manager{ + paths: paths.New(t.TempDir()), + destroyVGPU: func(context.Context, devices.VGPUAssignment) error { + t.Fatal("destroyVGPU must not be called for an ambiguous claim") + return nil + }, + } + require.NoError(t, m.ensureDirectories("ambiguous-claimant")) + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: "ambiguous-claimant", + GPUFramework: devices.VGPUFrameworkVendorVFIO, + GPUDevicePath: devicePath, + }})) + + stored := &StoredMetadata{ + Id: "requester", + GPUFramework: devices.VGPUFrameworkVendorVFIO, + GPUDevicePath: devicePath, + } + err := m.releaseStoredVGPU(context.Background(), stored) + require.Error(t, err) + assert.Contains(t, err.Error(), "ambiguous-claimant") + assert.Equal(t, devices.VGPUFrameworkVendorVFIO, stored.GPUFramework) + assert.Equal(t, devicePath, stored.GPUDevicePath) +} + func TestStoredVGPUDevicePath(t *testing.T) { t.Parallel() From 3af8301c0c9e098769a6cf5f4434cd886a1290ba Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Mon, 10 Aug 2026 14:14:21 +0000 Subject: [PATCH 19/76] Expose retained vGPU instance IDs --- cmd/api/api/instances.go | 6 ++++++ cmd/api/api/instances_test.go | 10 ++++++++++ 2 files changed, 16 insertions(+) diff --git a/cmd/api/api/instances.go b/cmd/api/api/instances.go index d62f1d7d2..8cfe72728 100644 --- a/cmd/api/api/instances.go +++ b/cmd/api/api/instances.go @@ -369,12 +369,18 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst case errors.As(err, &vgpuPending): log.ErrorContext(ctx, "failed to create instance", "error", err, "image", request.Body.Image) message := fmt.Sprintf("failed to create instance; vGPU release failed during rollback and instance %s retains the assignment, delete it to retry", vgpuPending.InstanceID) + innerCode := "vgpu_retained_instance" if !vgpuPending.Retained { message = fmt.Sprintf("failed to create instance; vGPU release failed during rollback and the retention record for instance %s could not be saved; the assignment is recovered on the next startup reconcile", vgpuPending.InstanceID) + innerCode = "vgpu_unretained_instance" } return oapi.CreateInstance500JSONResponse{ Code: "vgpu_cleanup_pending", Message: message, + InnerError: &oapi.ErrorDetail{ + Code: lo.ToPtr(innerCode), + Message: lo.ToPtr(vgpuPending.InstanceID), + }, }, nil case errors.Is(err, instances.ErrImageNotReady): return oapi.CreateInstance400JSONResponse{ diff --git a/cmd/api/api/instances_test.go b/cmd/api/api/instances_test.go index eec8e6dbb..a27334849 100644 --- a/cmd/api/api/instances_test.go +++ b/cmd/api/api/instances_test.go @@ -78,6 +78,11 @@ func TestCreateInstance_VGPUCleanupPendingBeatsWrappedErrorMapping(t *testing.T) assert.EqualValues(t, "vgpu_cleanup_pending", pending.Code) assert.Contains(t, pending.Message, "inst-1") assert.Contains(t, pending.Message, "delete it to retry") + require.NotNil(t, pending.InnerError) + require.NotNil(t, pending.InnerError.Code) + assert.Equal(t, "vgpu_retained_instance", *pending.InnerError.Code) + require.NotNil(t, pending.InnerError.Message) + assert.Equal(t, "inst-1", *pending.InnerError.Message) } func TestCreateInstance_VGPUCleanupPendingWithoutRetentionUsesReconcileGuidance(t *testing.T) { @@ -99,6 +104,11 @@ func TestCreateInstance_VGPUCleanupPendingWithoutRetentionUsesReconcileGuidance( assert.Contains(t, pending.Message, "retention record for instance inst-1 could not be saved") assert.Contains(t, pending.Message, "startup reconcile") assert.NotContains(t, pending.Message, "delete") + require.NotNil(t, pending.InnerError) + require.NotNil(t, pending.InnerError.Code) + assert.Equal(t, "vgpu_unretained_instance", *pending.InnerError.Code) + require.NotNil(t, pending.InnerError.Message) + assert.Equal(t, "inst-1", *pending.InnerError.Message) } func TestCreateInstance_AutoPullImage(t *testing.T) { From ad4c71388cc21547db92497412b60bab2b74eadf Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Mon, 10 Aug 2026 15:31:22 +0000 Subject: [PATCH 20/76] Harden vGPU startup rollback recovery --- cmd/api/main.go | 62 +++++++++++++++++++++++++++---------- cmd/api/main_test.go | 18 +++++++---- lib/instances/create.go | 4 +++ lib/instances/start.go | 25 +++------------ lib/instances/types.go | 3 +- lib/instances/vgpu.go | 27 +++++++++++++++- lib/instances/vgpu_test.go | 63 ++++++++++++++++++++++++++++++++++++-- 7 files changed, 155 insertions(+), 47 deletions(-) diff --git a/cmd/api/main.go b/cmd/api/main.go index 3374d9682..46308e201 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -185,26 +185,62 @@ func configureUFFDGraduationController(cfg *config.Config, instanceManager insta }, logger), nil } -func liveInstanceVGPUDevicePaths(ctx context.Context, instanceManager instances.Manager) (map[string]struct{}, error) { +const vgpuAssignmentStartupGracePeriod = 5 * time.Minute + +func liveInstanceVGPUDevicePaths(ctx context.Context, instanceManager instances.Manager) (map[string]struct{}, time.Duration, error) { allInstances, err := instanceManager.ListInstancesForReconcile(ctx) if err != nil { - return nil, err + return nil, 0, err } protected := make(map[string]struct{}) + var retryAfter time.Duration for _, inst := range allInstances { if inst.GPUDevicePath == "" { continue } - // A nil PID does not mean the assignment is orphaned: the PID is - // persisted only after the hypervisor starts, so a crash during boot - // leaves the device path without one. Only skip protection when the - // recorded hypervisor is known to be gone. - if inst.HypervisorPID != nil && !instances.HypervisorProcessExists(*inst.HypervisorPID, inst.SocketPath) { + if inst.HypervisorPID != nil { + if !instances.HypervisorProcessIdentityExists(*inst.HypervisorPID, inst.HypervisorStartTime, inst.SocketPath) { + continue + } + protected[inst.GPUDevicePath] = struct{}{} + continue + } + if inst.GPUAssignedAt == nil { + continue + } + remaining := vgpuAssignmentStartupGracePeriod - time.Since(*inst.GPUAssignedAt) + if remaining <= 0 { continue } protected[inst.GPUDevicePath] = struct{}{} + if retryAfter == 0 || remaining < retryAfter { + retryAfter = remaining + } } - return protected, nil + return protected, retryAfter, nil +} + +func reconcileVGPUs(ctx context.Context, instanceManager instances.Manager, logger *slog.Logger) { + protected, retryAfter, err := liveInstanceVGPUDevicePaths(ctx, instanceManager) + if err != nil { + logger.Warn("failed to list instances for vGPU reconcile protection; skipping vendor VFIO reconciliation", "error", err) + return + } + if err := devices.ReconcileVGPUs(ctx, protected); err != nil { + logger.Warn("failed to reconcile vGPU devices", "error", err) + } + if retryAfter <= 0 { + return + } + go func() { + timer := time.NewTimer(retryAfter) + defer timer.Stop() + select { + case <-ctx.Done(): + case <-timer.C: + reconcileVGPUs(ctx, instanceManager, logger) + } + }() } func run() error { @@ -408,15 +444,7 @@ func run() error { // Reconcile vGPU devices (clears orphaned vGPUs from previous runs) logger.Info("Reconciling vGPU devices...") - protected, err := liveInstanceVGPUDevicePaths(app.Ctx, app.InstanceManager) - if err != nil { - logger.Warn("failed to list instances for vGPU reconcile protection; skipping vendor VFIO reconciliation", "error", err) - protected = nil - } - if err := devices.ReconcileVGPUs(app.Ctx, protected); err != nil { - // Log but don't fail - vGPU cleanup is best-effort - logger.Warn("failed to reconcile vGPU devices", "error", err) - } + reconcileVGPUs(ctx, app.InstanceManager, logger) // Wire up resource validator for aggregate limit checking // This enables the instance manager to validate CPU, memory, network, and GPU diff --git a/cmd/api/main_test.go b/cmd/api/main_test.go index 573a404c1..02909e6ad 100644 --- a/cmd/api/main_test.go +++ b/cmd/api/main_test.go @@ -351,20 +351,26 @@ func (s vgpuReconcileManagerStub) ListInstancesForReconcile(context.Context) ([] return s.list, nil } -// The hypervisor PID is persisted only after boot, so an assignment without -// one may belong to a VM that is still starting and must stay protected. -func TestLiveInstanceVGPUDevicePathsProtectsAssignmentsWithoutPID(t *testing.T) { +func TestLiveInstanceVGPUDevicePathsBoundsProtectionWithoutPID(t *testing.T) { dead := exec.Command("true") require.NoError(t, dead.Run()) deadPID := dead.Process.Pid + recent := time.Now().Add(-time.Minute) + stale := time.Now().Add(-vgpuAssignmentStartupGracePeriod - time.Minute) manager := vgpuReconcileManagerStub{list: []instances.Instance{ - {StoredMetadata: instances.StoredMetadata{Id: "booting", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4"}}, - {StoredMetadata: instances.StoredMetadata{Id: "dead", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.5", HypervisorPID: &deadPID}}, + {StoredMetadata: instances.StoredMetadata{Id: "booting", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", GPUAssignedAt: &recent}}, + {StoredMetadata: instances.StoredMetadata{Id: "orphaned", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.5", GPUAssignedAt: &stale}}, + {StoredMetadata: instances.StoredMetadata{Id: "legacy", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.6"}}, + {StoredMetadata: instances.StoredMetadata{Id: "dead", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.7", HypervisorPID: &deadPID}}, }} - protected, err := liveInstanceVGPUDevicePaths(context.Background(), manager) + protected, retryAfter, err := liveInstanceVGPUDevicePaths(context.Background(), manager) require.NoError(t, err) + require.Positive(t, retryAfter) + require.LessOrEqual(t, retryAfter, vgpuAssignmentStartupGracePeriod) assert.Contains(t, protected, "/sys/bus/pci/devices/0000:82:00.4") assert.NotContains(t, protected, "/sys/bus/pci/devices/0000:82:00.5") + assert.NotContains(t, protected, "/sys/bus/pci/devices/0000:82:00.6") + assert.NotContains(t, protected, "/sys/bus/pci/devices/0000:82:00.7") } diff --git a/lib/instances/create.go b/lib/instances/create.go index 42fdcac39..98e1a6f29 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -277,6 +277,7 @@ func (m *manager) createInstance( var gpuFramework devices.VGPUFramework var gpuDevicePath string var gpuMdevUUID string + var gpuAssignedAt *time.Time var stored *StoredMetadata var retainedVGPU *StoredMetadata @@ -318,6 +319,8 @@ func (m *manager) createInstance( gpuFramework = gpuDevice.Framework gpuDevicePath = gpuDevice.SysfsPath gpuMdevUUID = gpuDevice.MdevUUID + assignedAt := m.nowUTC() + gpuAssignedAt = &assignedAt log.InfoContext(ctx, "created vGPU", "instance_id", id, "profile", gpuProfile, "uuid", gpuMdevUUID) // Add vGPU cleanup to stack @@ -426,6 +429,7 @@ func (m *manager) createInstance( GPUFramework: gpuFramework, GPUDevicePath: gpuDevicePath, GPUMdevUUID: gpuMdevUUID, + GPUAssignedAt: gpuAssignedAt, Entrypoint: req.Entrypoint, Cmd: req.Cmd, SkipKernelHeaders: req.SkipKernelHeaders, diff --git a/lib/instances/start.go b/lib/instances/start.go index b830bafb2..775038559 100644 --- a/lib/instances/start.go +++ b/lib/instances/start.go @@ -5,7 +5,6 @@ import ( "fmt" "time" - "github.com/kernel/hypeman/lib/devices" "github.com/kernel/hypeman/lib/egressproxy" "github.com/kernel/hypeman/lib/instances/phasetracking" "github.com/kernel/hypeman/lib/logger" @@ -64,6 +63,8 @@ func (m *manager) startInstance( } } + rollbackMeta := *meta + // 2a. Clear stale exit info from previous run and apply command overrides stored.ExitCode = nil stored.ExitMessage = "" @@ -168,28 +169,12 @@ func (m *manager) startInstance( log.ErrorContext(ctx, "failed to create vGPU", "instance_id", id, "profile", stored.GPUProfile, "error", err) return nil, fmt.Errorf("create vGPU for profile %s: %w", stored.GPUProfile, err) } - setStoredVGPUDevice(stored, device) + assignedAt := m.nowUTC() + setStoredVGPUDevice(stored, device, assignedAt) log.InfoContext(ctx, "created vGPU", "instance_id", id, "profile", stored.GPUProfile, "uuid", device.MdevUUID) // Add vGPU cleanup to stack cu.Add(func() { - log.DebugContext(ctx, "destroying vGPU on cleanup", "instance_id", id, "uuid", device.MdevUUID) - assignment := devices.VGPUAssignment{ - Framework: device.Framework, - DevicePath: device.SysfsPath, - MdevUUID: device.MdevUUID, - InstanceID: id, - } - if err := m.destroyVGPUAssignment(ctx, assignment); err != nil { - log.WarnContext(ctx, "failed to destroy vGPU on cleanup", "instance_id", id, "uuid", device.MdevUUID, "error", err) - if saveErr := m.saveMetadata(meta); saveErr != nil { - log.ErrorContext(ctx, "failed to retain vGPU assignment metadata after cleanup failure", "instance_id", id, "error", saveErr) - } - } else { - clearStoredVGPUDevice(stored) - if saveErr := m.saveMetadata(meta); saveErr != nil { - log.ErrorContext(ctx, "failed to save metadata after vGPU cleanup", "instance_id", id, "error", saveErr) - } - } + m.cleanupStartVGPU(ctx, id, device, assignedAt, rollbackMeta) }) if err := m.saveMetadata(meta); err != nil { log.ErrorContext(ctx, "failed to save metadata after vGPU creation", "instance_id", id, "error", err) diff --git a/lib/instances/types.go b/lib/instances/types.go index 6aac15985..a264498fd 100644 --- a/lib/instances/types.go +++ b/lib/instances/types.go @@ -154,7 +154,8 @@ type StoredMetadata struct { GPUProfile string // vGPU profile name (e.g., "L40S-1Q") GPUFramework devices.VGPUFramework GPUDevicePath string - GPUMdevUUID string // populated for mdev-backed vGPUs + GPUMdevUUID string // populated for mdev-backed vGPUs + GPUAssignedAt *time.Time // set before hypervisor startup to bound crash recovery protection // Command overrides (like docker run ) Entrypoint []string // Override image entrypoint (nil = use image default) diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index 9bca44dbc..3e7dc350c 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -3,6 +3,7 @@ package instances import ( "context" "path/filepath" + "time" "github.com/kernel/hypeman/lib/devices" "github.com/kernel/hypeman/lib/logger" @@ -49,16 +50,40 @@ func (m *manager) destroyVGPUAssignment(ctx context.Context, assignment devices. return destroy(ctx, assignment) } -func setStoredVGPUDevice(stored *StoredMetadata, device *devices.VGPUDevice) { +func setStoredVGPUDevice(stored *StoredMetadata, device *devices.VGPUDevice, assignedAt time.Time) { stored.GPUFramework = device.Framework stored.GPUDevicePath = device.SysfsPath stored.GPUMdevUUID = device.MdevUUID + stored.GPUAssignedAt = &assignedAt } func clearStoredVGPUDevice(stored *StoredMetadata) { stored.GPUFramework = devices.VGPUFrameworkNone stored.GPUDevicePath = "" stored.GPUMdevUUID = "" + stored.GPUAssignedAt = nil +} + +func (m *manager) cleanupStartVGPU(ctx context.Context, instanceID string, device *devices.VGPUDevice, assignedAt time.Time, rollbackMeta metadata) { + assignment := devices.VGPUAssignment{ + Framework: device.Framework, + DevicePath: device.SysfsPath, + MdevUUID: device.MdevUUID, + InstanceID: instanceID, + } + cleanupMeta := rollbackMeta + releaseErr := m.destroyVGPUAssignment(ctx, assignment) + if releaseErr != nil { + logger.FromContext(ctx).WarnContext(ctx, "failed to destroy vGPU on cleanup", "instance_id", instanceID, "error", releaseErr) + setStoredVGPUDevice(&cleanupMeta.StoredMetadata, device, assignedAt) + } + if err := m.saveMetadata(&cleanupMeta); err != nil { + message := "failed to save metadata after vGPU cleanup" + if releaseErr != nil { + message = "failed to retain vGPU assignment metadata after cleanup failure" + } + logger.FromContext(ctx).ErrorContext(ctx, message, "instance_id", instanceID, "error", err) + } } func (m *manager) releaseStoredVGPU(ctx context.Context, stored *StoredMetadata) error { diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index d4e0c6489..366e343f2 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -7,6 +7,7 @@ import ( "path/filepath" "sync" "testing" + "time" "github.com/kernel/hypeman/lib/devices" "github.com/kernel/hypeman/lib/hypervisor" @@ -188,13 +189,68 @@ func TestStartRollbackRetainsVGPUAssignmentAfterFailedDestroy(t *testing.T) { }) t.Setenv("TMPDIR", filepath.Join(t.TempDir(), "missing")) - _, err := m.startInstance(context.Background(), id, StartInstanceRequest{}) + _, err := m.startInstance(context.Background(), id, StartInstanceRequest{Entrypoint: []string{"new-entrypoint"}}) require.Error(t, err) stored, err := m.loadMetadata(id) require.NoError(t, err) assert.Equal(t, devices.VGPUFrameworkVendorVFIO, stored.GPUFramework) assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", stored.GPUDevicePath) + assert.NotNil(t, stored.GPUAssignedAt) + assert.Empty(t, stored.Entrypoint) +} + +func TestCleanupStartVGPURestoresMetadataAfterBootFailure(t *testing.T) { + m := &manager{ + paths: paths.New(t.TempDir()), + destroyVGPU: func(context.Context, devices.VGPUAssignment) error { + return nil + }, + } + const id = "failed-start" + require.NoError(t, m.ensureDirectories(id)) + + previousStart := time.Now().Add(-time.Hour).UTC() + previousProgramStart := previousStart.Add(time.Second) + exitCode := 1 + rollbackMeta := metadata{StoredMetadata: StoredMetadata{ + Id: id, + GPUProfile: "NVIDIA L40S-2Q", + Entrypoint: []string{"old-entrypoint"}, + Cmd: []string{"old-command"}, + StartedAt: &previousStart, + ProgramStartedAt: &previousProgramStart, + ExitCode: &exitCode, + ExitMessage: "previous exit", + }} + + partial := rollbackMeta + partial.Entrypoint = []string{"new-entrypoint"} + partial.Cmd = []string{"new-command"} + partial.StartedAt = ptr(time.Now().UTC()) + partial.ProgramStartedAt = nil + partial.ExitCode = nil + partial.ExitMessage = "" + assignedAt := time.Now().UTC() + device := &devices.VGPUDevice{ + Framework: devices.VGPUFrameworkVendorVFIO, + SysfsPath: "/sys/bus/pci/devices/0000:82:00.4", + } + setStoredVGPUDevice(&partial.StoredMetadata, device, assignedAt) + require.NoError(t, m.saveMetadata(&partial)) + + m.cleanupStartVGPU(context.Background(), id, device, assignedAt, rollbackMeta) + + stored, err := m.loadMetadata(id) + require.NoError(t, err) + assert.Equal(t, rollbackMeta.Entrypoint, stored.Entrypoint) + assert.Equal(t, rollbackMeta.Cmd, stored.Cmd) + assert.Equal(t, rollbackMeta.StartedAt, stored.StartedAt) + assert.Equal(t, rollbackMeta.ProgramStartedAt, stored.ProgramStartedAt) + assert.Equal(t, rollbackMeta.ExitCode, stored.ExitCode) + assert.Equal(t, rollbackMeta.ExitMessage, stored.ExitMessage) + assert.Empty(t, stored.GPUDevicePath) + assert.Nil(t, stored.GPUAssignedAt) } func TestVGPUAssignmentClaimedByLiveInstanceFailsOnInvalidMetadata(t *testing.T) { @@ -341,16 +397,19 @@ func TestReleaseStoredVGPURetainsMetadataOnFailure(t *testing.T) { func TestSetAndClearStoredVGPUDevice(t *testing.T) { t.Parallel() + assignedAt := time.Now().UTC() stored := &StoredMetadata{} setStoredVGPUDevice(stored, &devices.VGPUDevice{ Framework: devices.VGPUFrameworkVendorVFIO, SysfsPath: "/sys/bus/pci/devices/0000:82:00.4", - }) + }, assignedAt) assert.Equal(t, devices.VGPUFrameworkVendorVFIO, stored.GPUFramework) assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", stored.GPUDevicePath) + assert.Equal(t, assignedAt, *stored.GPUAssignedAt) clearStoredVGPUDevice(stored) assert.Empty(t, stored.GPUFramework) assert.Empty(t, stored.GPUDevicePath) assert.Empty(t, stored.GPUMdevUUID) + assert.Nil(t, stored.GPUAssignedAt) } From df3dcf8d64ae9cc22e96cb6a3d7bd1ed0572c8ef Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Mon, 10 Aug 2026 16:12:47 +0000 Subject: [PATCH 21/76] Preserve the create failure cause in vGPU cleanup errors The vgpu_cleanup_pending response replaced the original create error with cleanup guidance, leaving the cause only in server logs. Prefix the message with the wrapped error so callers see why creation failed as well as how to recover. --- cmd/api/api/instances.go | 4 ++-- cmd/api/api/instances_test.go | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/cmd/api/api/instances.go b/cmd/api/api/instances.go index 8cfe72728..681be6ab1 100644 --- a/cmd/api/api/instances.go +++ b/cmd/api/api/instances.go @@ -368,10 +368,10 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst // errors.Is case would match the cause and hide the pending vGPU cleanup. case errors.As(err, &vgpuPending): log.ErrorContext(ctx, "failed to create instance", "error", err, "image", request.Body.Image) - message := fmt.Sprintf("failed to create instance; vGPU release failed during rollback and instance %s retains the assignment, delete it to retry", vgpuPending.InstanceID) + message := fmt.Sprintf("failed to create instance: %v; vGPU release failed during rollback and instance %s retains the assignment, delete it to retry", vgpuPending.Err, vgpuPending.InstanceID) innerCode := "vgpu_retained_instance" if !vgpuPending.Retained { - message = fmt.Sprintf("failed to create instance; vGPU release failed during rollback and the retention record for instance %s could not be saved; the assignment is recovered on the next startup reconcile", vgpuPending.InstanceID) + message = fmt.Sprintf("failed to create instance: %v; vGPU release failed during rollback and the retention record for instance %s could not be saved; the assignment is recovered on the next startup reconcile", vgpuPending.Err, vgpuPending.InstanceID) innerCode = "vgpu_unretained_instance" } return oapi.CreateInstance500JSONResponse{ diff --git a/cmd/api/api/instances_test.go b/cmd/api/api/instances_test.go index a27334849..fdd444f8e 100644 --- a/cmd/api/api/instances_test.go +++ b/cmd/api/api/instances_test.go @@ -77,6 +77,8 @@ func TestCreateInstance_VGPUCleanupPendingBeatsWrappedErrorMapping(t *testing.T) require.True(t, ok, "expected 500 vgpu_cleanup_pending, got %T", resp) assert.EqualValues(t, "vgpu_cleanup_pending", pending.Code) assert.Contains(t, pending.Message, "inst-1") + assert.Contains(t, pending.Message, network.ErrNameExists.Error(), + "the underlying create failure must survive the cleanup guidance") assert.Contains(t, pending.Message, "delete it to retry") require.NotNil(t, pending.InnerError) require.NotNil(t, pending.InnerError.Code) @@ -102,6 +104,8 @@ func TestCreateInstance_VGPUCleanupPendingWithoutRetentionUsesReconcileGuidance( require.True(t, ok, "expected 500 vgpu_cleanup_pending, got %T", resp) assert.EqualValues(t, "vgpu_cleanup_pending", pending.Code) assert.Contains(t, pending.Message, "retention record for instance inst-1 could not be saved") + assert.Contains(t, pending.Message, network.ErrNameExists.Error(), + "the underlying create failure must survive the cleanup guidance") assert.Contains(t, pending.Message, "startup reconcile") assert.NotContains(t, pending.Message, "delete") require.NotNil(t, pending.InnerError) From d60193ed3a5ac051ba66b187ef3763d5b326d10b Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Mon, 10 Aug 2026 17:33:44 +0000 Subject: [PATCH 22/76] Fix vGPU reconciliation edge cases --- cmd/api/main.go | 9 ++++----- cmd/api/main_test.go | 4 ++-- lib/instances/create.go | 2 ++ lib/instances/vgpu.go | 16 ++++++++++++---- lib/instances/vgpu_test.go | 27 ++++++++++++++++++++++++++- 5 files changed, 46 insertions(+), 12 deletions(-) diff --git a/cmd/api/main.go b/cmd/api/main.go index 46308e201..eeb6e9fae 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -185,8 +185,6 @@ func configureUFFDGraduationController(cfg *config.Config, instanceManager insta }, logger), nil } -const vgpuAssignmentStartupGracePeriod = 5 * time.Minute - func liveInstanceVGPUDevicePaths(ctx context.Context, instanceManager instances.Manager) (map[string]struct{}, time.Duration, error) { allInstances, err := instanceManager.ListInstancesForReconcile(ctx) if err != nil { @@ -208,7 +206,7 @@ func liveInstanceVGPUDevicePaths(ctx context.Context, instanceManager instances. if inst.GPUAssignedAt == nil { continue } - remaining := vgpuAssignmentStartupGracePeriod - time.Since(*inst.GPUAssignedAt) + remaining := instances.VGPUAssignmentStartupGracePeriod - time.Since(*inst.GPUAssignedAt) if remaining <= 0 { continue } @@ -223,8 +221,9 @@ func liveInstanceVGPUDevicePaths(ctx context.Context, instanceManager instances. func reconcileVGPUs(ctx context.Context, instanceManager instances.Manager, logger *slog.Logger) { protected, retryAfter, err := liveInstanceVGPUDevicePaths(ctx, instanceManager) if err != nil { - logger.Warn("failed to list instances for vGPU reconcile protection; skipping vendor VFIO reconciliation", "error", err) - return + logger.Warn("failed to list instances for vGPU reconcile protection; reconciling mdev only", "error", err) + protected = nil + retryAfter = 0 } if err := devices.ReconcileVGPUs(ctx, protected); err != nil { logger.Warn("failed to reconcile vGPU devices", "error", err) diff --git a/cmd/api/main_test.go b/cmd/api/main_test.go index 02909e6ad..09a4a2419 100644 --- a/cmd/api/main_test.go +++ b/cmd/api/main_test.go @@ -356,7 +356,7 @@ func TestLiveInstanceVGPUDevicePathsBoundsProtectionWithoutPID(t *testing.T) { require.NoError(t, dead.Run()) deadPID := dead.Process.Pid recent := time.Now().Add(-time.Minute) - stale := time.Now().Add(-vgpuAssignmentStartupGracePeriod - time.Minute) + stale := time.Now().Add(-instances.VGPUAssignmentStartupGracePeriod - time.Minute) manager := vgpuReconcileManagerStub{list: []instances.Instance{ {StoredMetadata: instances.StoredMetadata{Id: "booting", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", GPUAssignedAt: &recent}}, @@ -368,7 +368,7 @@ func TestLiveInstanceVGPUDevicePathsBoundsProtectionWithoutPID(t *testing.T) { protected, retryAfter, err := liveInstanceVGPUDevicePaths(context.Background(), manager) require.NoError(t, err) require.Positive(t, retryAfter) - require.LessOrEqual(t, retryAfter, vgpuAssignmentStartupGracePeriod) + require.LessOrEqual(t, retryAfter, instances.VGPUAssignmentStartupGracePeriod) assert.Contains(t, protected, "/sys/bus/pci/devices/0000:82:00.4") assert.NotContains(t, protected, "/sys/bus/pci/devices/0000:82:00.5") assert.NotContains(t, protected, "/sys/bus/pci/devices/0000:82:00.6") diff --git a/lib/instances/create.go b/lib/instances/create.go index 98e1a6f29..6ea250e55 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -351,6 +351,7 @@ func (m *manager) createInstance( GPUFramework: gpuDevice.Framework, GPUDevicePath: gpuDevice.SysfsPath, GPUMdevUUID: gpuDevice.MdevUUID, + GPUAssignedAt: gpuAssignedAt, } } } @@ -672,6 +673,7 @@ func (m *manager) cleanupFailedCreate(ctx context.Context, id string, retainedVG GPUFramework: retainedVGPU.GPUFramework, GPUDevicePath: retainedVGPU.GPUDevicePath, GPUMdevUUID: retainedVGPU.GPUMdevUUID, + GPUAssignedAt: retainedVGPU.GPUAssignedAt, } if err := m.saveMetadata(&metadata{StoredMetadata: retained}); err != nil { log.ErrorContext(ctx, "failed to retain vGPU assignment metadata after cleanup failure", "instance_id", id, "error", err) diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index 3e7dc350c..0d12e5323 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -6,6 +6,7 @@ import ( "time" "github.com/kernel/hypeman/lib/devices" + "github.com/kernel/hypeman/lib/hypervisor" "github.com/kernel/hypeman/lib/logger" ) @@ -16,6 +17,10 @@ func validateVGPUHypervisor(hvType hypervisor.Type) error { return nil } +// VGPUAssignmentStartupGracePeriod bounds how long an assignment without a +// persisted hypervisor PID is treated as potentially live. +const VGPUAssignmentStartupGracePeriod = 5 * time.Minute + // VGPUCleanupPendingError reports a failed create whose vGPU release also // failed during rollback. When Retained is true, deleting the retained instance // retries the release; otherwise startup reconciliation recovers the assignment. @@ -125,9 +130,9 @@ func (m *manager) releaseStoredVGPU(ctx context.Context, stored *StoredMetadata) // stored metadata claims devicePath. It reads raw metadata instead of // hydrating full instances: the scan runs on every vendor VFIO release, and // deriving state would query the hypervisor of every instance on the host. -// A confirmed live claimant returns true. Unreadable metadata, a missing PID, -// or unverifiable process ownership returns an error so the requester retains -// its assignment for a later retry. +// A confirmed live claimant returns true. Unreadable metadata, a recent +// assignment without a PID, or unverifiable process ownership returns an error +// so the requester retains its assignment for a later retry. func (m *manager) vgpuAssignmentClaimedByLiveInstance(ctx context.Context, excludeID, devicePath string) (bool, error) { files, err := m.listMetadataFilesWithStatErrors(true) if err != nil { @@ -147,7 +152,10 @@ func (m *manager) vgpuAssignmentClaimedByLiveInstance(ctx context.Context, exclu continue } if stored.HypervisorPID == nil { - return false, fmt.Errorf("cannot confirm liveness of vGPU claimant %s on %s: no persisted hypervisor PID", id, devicePath) + if stored.GPUAssignedAt == nil || time.Since(*stored.GPUAssignedAt) >= VGPUAssignmentStartupGracePeriod { + continue + } + return false, fmt.Errorf("cannot confirm liveness of recent vGPU claimant %s on %s: no persisted hypervisor PID", id, devicePath) } pid, err := resolveLiveHypervisorPID(stored.HypervisorPID, stored.HypervisorStartTime, stored.SocketPath) if err != nil { diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index 366e343f2..7eed17444 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -27,6 +27,7 @@ func TestCleanupFailedCreateRetainsVGPUAssignment(t *testing.T) { t.Parallel() m := &manager{paths: paths.New(t.TempDir())} + assignedAt := time.Now().UTC() stored := &StoredMetadata{ Id: "failed-create", Name: "failed-create", @@ -34,6 +35,7 @@ func TestCleanupFailedCreateRetainsVGPUAssignment(t *testing.T) { GPUFramework: devices.VGPUFrameworkVendorVFIO, GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", GPUMdevUUID: "mdev-uuid", + GPUAssignedAt: &assignedAt, NetworkEnabled: true, IP: "192.0.2.1", Volumes: []VolumeAttachment{{VolumeID: "volume"}}, @@ -49,6 +51,7 @@ func TestCleanupFailedCreateRetainsVGPUAssignment(t *testing.T) { assert.Equal(t, stored.GPUFramework, retained.GPUFramework) assert.Equal(t, stored.GPUDevicePath, retained.GPUDevicePath) assert.Equal(t, stored.GPUMdevUUID, retained.GPUMdevUUID) + assert.Equal(t, stored.GPUAssignedAt, retained.GPUAssignedAt) assert.Empty(t, retained.Name) assert.Empty(t, retained.GPUProfile) assert.False(t, retained.NetworkEnabled) @@ -282,15 +285,17 @@ func TestVGPUAssignmentClaimedByLiveInstanceNormalizesLegacyMdevPath(t *testing. assert.True(t, claimed) } -func TestVGPUAssignmentClaimedByLiveInstanceErrorsOnNilPIDClaim(t *testing.T) { +func TestVGPUAssignmentClaimedByLiveInstanceErrorsOnRecentNilPIDClaim(t *testing.T) { t.Parallel() m := &manager{paths: paths.New(t.TempDir())} require.NoError(t, m.ensureDirectories("booting-claimant")) + assignedAt := time.Now().UTC() require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ Id: "booting-claimant", Name: "booting-claimant", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", + GPUAssignedAt: &assignedAt, }})) _, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "other-instance", "/sys/bus/pci/devices/0000:82:00.4") @@ -298,6 +303,24 @@ func TestVGPUAssignmentClaimedByLiveInstanceErrorsOnNilPIDClaim(t *testing.T) { assert.Contains(t, err.Error(), "booting-claimant") } +func TestVGPUAssignmentClaimedByLiveInstanceIgnoresStaleNilPIDClaim(t *testing.T) { + t.Parallel() + + m := &manager{paths: paths.New(t.TempDir())} + require.NoError(t, m.ensureDirectories("stale-claimant")) + assignedAt := time.Now().Add(-VGPUAssignmentStartupGracePeriod - time.Minute) + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: "stale-claimant", + Name: "stale-claimant", + GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", + GPUAssignedAt: &assignedAt, + }})) + + claimed, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "other-instance", "/sys/bus/pci/devices/0000:82:00.4") + require.NoError(t, err) + assert.False(t, claimed) +} + func TestVGPUAssignmentClaimedByLiveInstanceIgnoresDeadClaim(t *testing.T) { t.Parallel() @@ -349,10 +372,12 @@ func TestReleaseStoredVGPURetainsRequesterOnAmbiguousClaim(t *testing.T) { }, } require.NoError(t, m.ensureDirectories("ambiguous-claimant")) + assignedAt := time.Now().UTC() require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ Id: "ambiguous-claimant", GPUFramework: devices.VGPUFrameworkVendorVFIO, GPUDevicePath: devicePath, + GPUAssignedAt: &assignedAt, }})) stored := &StoredMetadata{ From d0de0ccbc3e89460c7b52415fa7e46cc25ee5006 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Mon, 10 Aug 2026 18:26:01 +0000 Subject: [PATCH 23/76] Use boot-scoped hypervisor identities for vGPUs --- cmd/api/main.go | 2 +- lib/instances/vgpu.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/api/main.go b/cmd/api/main.go index eeb6e9fae..026267a05 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -197,7 +197,7 @@ func liveInstanceVGPUDevicePaths(ctx context.Context, instanceManager instances. continue } if inst.HypervisorPID != nil { - if !instances.HypervisorProcessIdentityExists(*inst.HypervisorPID, inst.HypervisorStartTime, inst.SocketPath) { + if !instances.HypervisorProcessIdentityExists(*inst.HypervisorPID, inst.HypervisorStartTime, inst.HypervisorBootID, inst.SocketPath) { continue } protected[inst.GPUDevicePath] = struct{}{} diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index 0d12e5323..772d93e17 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -157,7 +157,7 @@ func (m *manager) vgpuAssignmentClaimedByLiveInstance(ctx context.Context, exclu } return false, fmt.Errorf("cannot confirm liveness of recent vGPU claimant %s on %s: no persisted hypervisor PID", id, devicePath) } - pid, err := resolveLiveHypervisorPID(stored.HypervisorPID, stored.HypervisorStartTime, stored.SocketPath) + pid, err := resolveLiveHypervisorPID(stored.HypervisorPID, stored.HypervisorStartTime, stored.HypervisorBootID, stored.SocketPath) if err != nil { return false, fmt.Errorf("cannot confirm liveness of vGPU claimant %s on %s: %w", id, devicePath, err) } From 5b1b190eb83d2167706629ab897e942d585d3a29 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Mon, 10 Aug 2026 18:34:19 +0000 Subject: [PATCH 24/76] Run vGPU rollback tests with QEMU --- lib/instances/vgpu_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index 7eed17444..56c6f8574 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -153,7 +153,7 @@ func newStartRollbackVGPUManager(t *testing.T, destroy func(context.Context, dev Name: id, Image: "test-image", GPUProfile: "NVIDIA L40S-2Q", - HypervisorType: lifecycleNoopHypervisorType, + HypervisorType: hypervisor.TypeQEMU, SocketPath: m.paths.InstanceSocket(id, "noop.sock"), DataDir: m.paths.InstanceDir(id), }})) From 5420fb9be464d4a7f76d00d1628e20326950b362 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Mon, 10 Aug 2026 19:07:05 +0000 Subject: [PATCH 25/76] Protect new vGPU assignments from stale PIDs --- cmd/api/main.go | 5 +---- cmd/api/main_test.go | 4 +++- lib/instances/create.go | 6 ++++++ lib/instances/start.go | 9 +++++++++ lib/instances/vgpu.go | 1 + lib/instances/vgpu_test.go | 12 +++++++++++- 6 files changed, 31 insertions(+), 6 deletions(-) diff --git a/cmd/api/main.go b/cmd/api/main.go index 026267a05..982e15c3f 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -196,10 +196,7 @@ func liveInstanceVGPUDevicePaths(ctx context.Context, instanceManager instances. if inst.GPUDevicePath == "" { continue } - if inst.HypervisorPID != nil { - if !instances.HypervisorProcessIdentityExists(*inst.HypervisorPID, inst.HypervisorStartTime, inst.HypervisorBootID, inst.SocketPath) { - continue - } + if inst.HypervisorPID != nil && instances.HypervisorProcessIdentityExists(*inst.HypervisorPID, inst.HypervisorStartTime, inst.HypervisorBootID, inst.SocketPath) { protected[inst.GPUDevicePath] = struct{}{} continue } diff --git a/cmd/api/main_test.go b/cmd/api/main_test.go index 09a4a2419..217e9db8d 100644 --- a/cmd/api/main_test.go +++ b/cmd/api/main_test.go @@ -351,7 +351,7 @@ func (s vgpuReconcileManagerStub) ListInstancesForReconcile(context.Context) ([] return s.list, nil } -func TestLiveInstanceVGPUDevicePathsBoundsProtectionWithoutPID(t *testing.T) { +func TestLiveInstanceVGPUDevicePathsBoundsStartupProtection(t *testing.T) { dead := exec.Command("true") require.NoError(t, dead.Run()) deadPID := dead.Process.Pid @@ -363,6 +363,7 @@ func TestLiveInstanceVGPUDevicePathsBoundsProtectionWithoutPID(t *testing.T) { {StoredMetadata: instances.StoredMetadata{Id: "orphaned", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.5", GPUAssignedAt: &stale}}, {StoredMetadata: instances.StoredMetadata{Id: "legacy", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.6"}}, {StoredMetadata: instances.StoredMetadata{Id: "dead", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.7", HypervisorPID: &deadPID}}, + {StoredMetadata: instances.StoredMetadata{Id: "stale-pid-booting", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.8", HypervisorPID: &deadPID, GPUAssignedAt: &recent}}, }} protected, retryAfter, err := liveInstanceVGPUDevicePaths(context.Background(), manager) @@ -373,4 +374,5 @@ func TestLiveInstanceVGPUDevicePathsBoundsProtectionWithoutPID(t *testing.T) { assert.NotContains(t, protected, "/sys/bus/pci/devices/0000:82:00.5") assert.NotContains(t, protected, "/sys/bus/pci/devices/0000:82:00.6") assert.NotContains(t, protected, "/sys/bus/pci/devices/0000:82:00.7") + assert.Contains(t, protected, "/sys/bus/pci/devices/0000:82:00.8") } diff --git a/lib/instances/create.go b/lib/instances/create.go index 6ea250e55..0e8ef4ce7 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -106,6 +106,11 @@ func (m *manager) createInstance( if hvType == "" { hvType = m.defaultHypervisor } + if req.GPU != nil && req.GPU.Profile != "" { + if err := validateVGPUHypervisor(hvType); err != nil { + return nil, fmt.Errorf("%w: %w", ErrInvalidRequest, err) + } + } starter, starterErr := m.getVMStarter(hvType) if starterErr == nil { if err := m.validateCreateVMConfig(starter, req, hvType); err != nil { @@ -113,6 +118,7 @@ func (m *manager) createInstance( } } + // 2. Validate image exists and is ready; auto-pull if not found log.DebugContext(ctx, "validating image", "image", req.Image) imageCtx, imageSpanEnd := m.startLifecycleStep(ctx, "resolve_image", diff --git a/lib/instances/start.go b/lib/instances/start.go index 775038559..ecb33f11f 100644 --- a/lib/instances/start.go +++ b/lib/instances/start.go @@ -48,6 +48,11 @@ func (m *manager) startInstance( return nil, fmt.Errorf("%w: cannot start from state %s, must be Stopped", ErrInvalidState, inst.State) } + if stored.GPUProfile != "" { + if err := validateVGPUHypervisor(stored.HypervisorType); err != nil { + return nil, fmt.Errorf("%w: %w", ErrInvalidState, err) + } + } // Release any assignment retained by an earlier failed release and // persist the cleared fields immediately, so a failure later in start // cannot leave on-disk metadata pointing at a device that is already @@ -63,6 +68,10 @@ func (m *manager) startInstance( } } + // Do not persist the previous VMM's identity with a new vGPU assignment. + stored.HypervisorPID = nil + stored.HypervisorStartTime = 0 + stored.HypervisorBootID = "" rollbackMeta := *meta // 2a. Clear stale exit info from previous run and apply command overrides diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index 772d93e17..6fd02352a 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -2,6 +2,7 @@ package instances import ( "context" + "fmt" "path/filepath" "time" diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index 56c6f8574..4491bdd17 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -190,9 +190,16 @@ func TestStartRollbackRetainsVGPUAssignmentAfterFailedDestroy(t *testing.T) { m, id := newStartRollbackVGPUManager(t, func(context.Context, devices.VGPUAssignment) error { return errors.New("destroy failed") }) + meta, err := m.loadMetadata(id) + require.NoError(t, err) + stalePID := os.Getpid() + meta.HypervisorPID = &stalePID + meta.HypervisorStartTime = 1 + meta.HypervisorBootID = "previous-boot" + require.NoError(t, m.saveMetadata(meta)) t.Setenv("TMPDIR", filepath.Join(t.TempDir(), "missing")) - _, err := m.startInstance(context.Background(), id, StartInstanceRequest{Entrypoint: []string{"new-entrypoint"}}) + _, err = m.startInstance(context.Background(), id, StartInstanceRequest{Entrypoint: []string{"new-entrypoint"}}) require.Error(t, err) stored, err := m.loadMetadata(id) @@ -200,6 +207,9 @@ func TestStartRollbackRetainsVGPUAssignmentAfterFailedDestroy(t *testing.T) { assert.Equal(t, devices.VGPUFrameworkVendorVFIO, stored.GPUFramework) assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", stored.GPUDevicePath) assert.NotNil(t, stored.GPUAssignedAt) + assert.Nil(t, stored.HypervisorPID) + assert.Zero(t, stored.HypervisorStartTime) + assert.Empty(t, stored.HypervisorBootID) assert.Empty(t, stored.Entrypoint) } From 83c7f5167c0dc8542b5afeb6e3c9e275d428d379 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Mon, 10 Aug 2026 19:30:28 +0000 Subject: [PATCH 26/76] Persist vGPU assignments after create rollback failure --- lib/instances/create.go | 1 + lib/instances/start.go | 8 ++++++ lib/instances/vgpu.go | 23 ++++++++++++++++ lib/instances/vgpu_test.go | 56 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 88 insertions(+) diff --git a/lib/instances/create.go b/lib/instances/create.go index 0e8ef4ce7..c221f698a 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -318,6 +318,7 @@ func (m *manager) createInstance( log.InfoContext(ctx, "creating vGPU", "instance_id", id, "profile", req.GPU.Profile) gpuDevice, err = m.createVGPUDevice(ctx, req.GPU.Profile, id) if err != nil { + retainedVGPU = retainedVGPUFromCreateError(id, m.nowUTC(), err) log.ErrorContext(ctx, "failed to create vGPU", "profile", req.GPU.Profile, "error", err) return nil, wrapCreateVGPUErr(req.GPU.Profile, err) } diff --git a/lib/instances/start.go b/lib/instances/start.go index ecb33f11f..39418ad7d 100644 --- a/lib/instances/start.go +++ b/lib/instances/start.go @@ -175,6 +175,14 @@ func (m *manager) startInstance( log.InfoContext(ctx, "creating vGPU for start", "instance_id", id, "profile", stored.GPUProfile) device, err := m.createVGPUDevice(ctx, stored.GPUProfile, id) if err != nil { + if pendingDevice, ok := vgpuDevicePendingCleanup(err); ok { + assignedAt := m.nowUTC() + setStoredVGPUDevice(stored, pendingDevice, assignedAt) + if saveErr := m.saveMetadata(meta); saveErr != nil { + log.ErrorContext(ctx, "failed to retain vGPU assignment after create rollback failure", "instance_id", id, "error", saveErr) + return nil, fmt.Errorf("create vGPU for profile %s: %w; retain assignment: %v", stored.GPUProfile, err, saveErr) + } + } log.ErrorContext(ctx, "failed to create vGPU", "instance_id", id, "profile", stored.GPUProfile, "error", err) return nil, fmt.Errorf("create vGPU for profile %s: %w", stored.GPUProfile, err) } diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index 6fd02352a..a52cfa5d1 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -2,6 +2,7 @@ package instances import ( "context" + "errors" "fmt" "path/filepath" "time" @@ -48,6 +49,28 @@ func (m *manager) createVGPUDevice(ctx context.Context, profileName, instanceID return create(ctx, profileName, instanceID) } +func vgpuDevicePendingCleanup(err error) (*devices.VGPUDevice, bool) { + var pending *devices.VGPUCreateCleanupPendingError + if !errors.As(err, &pending) { + return nil, false + } + return &pending.Device, true +} + +func retainedVGPUFromCreateError(instanceID string, assignedAt time.Time, err error) *StoredMetadata { + device, ok := vgpuDevicePendingCleanup(err) + if !ok { + return nil + } + return &StoredMetadata{ + Id: instanceID, + GPUFramework: device.Framework, + GPUDevicePath: device.SysfsPath, + GPUMdevUUID: device.MdevUUID, + GPUAssignedAt: &assignedAt, + } +} + func (m *manager) destroyVGPUAssignment(ctx context.Context, assignment devices.VGPUAssignment) error { destroy := m.destroyVGPU if destroy == nil { diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index 4491bdd17..3cfd04430 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -3,6 +3,7 @@ package instances import ( "context" "errors" + "fmt" "os" "path/filepath" "sync" @@ -128,6 +129,35 @@ func TestVGPUCleanupPendingErrorUnwraps(t *testing.T) { assert.Equal(t, "boot failed; vGPU release failed during rollback and the retention record for instance inst-1 could not be saved; the assignment is recovered on the next startup reconcile", unpersisted.Error()) } +func TestVGPUDevicePendingCleanup(t *testing.T) { + t.Parallel() + + device := devices.VGPUDevice{ + Framework: devices.VGPUFrameworkVendorVFIO, + SysfsPath: "/sys/bus/pci/devices/0000:82:00.4", + } + cause := errors.New("rollback failed") + pending := &devices.VGPUCreateCleanupPendingError{Device: device, Err: cause} + + wrapped := fmt.Errorf("create failed: %w", pending) + actual, ok := vgpuDevicePendingCleanup(wrapped) + require.True(t, ok) + assert.Equal(t, device, *actual) + + assignedAt := time.Now().UTC() + retained := retainedVGPUFromCreateError("inst-1", assignedAt, wrapped) + require.NotNil(t, retained) + assert.Equal(t, "inst-1", retained.Id) + assert.Equal(t, device.Framework, retained.GPUFramework) + assert.Equal(t, device.SysfsPath, retained.GPUDevicePath) + assert.Equal(t, assignedAt, *retained.GPUAssignedAt) + + actual, ok = vgpuDevicePendingCleanup(cause) + assert.False(t, ok) + assert.Nil(t, actual) + assert.Nil(t, retainedVGPUFromCreateError("inst-1", assignedAt, cause)) +} + func newStartRollbackVGPUManager(t *testing.T, destroy func(context.Context, devices.VGPUAssignment) error) (*manager, string) { t.Helper() m := &manager{ @@ -160,6 +190,32 @@ func newStartRollbackVGPUManager(t *testing.T, destroy func(context.Context, dev return m, id } +func TestStartRetainsVGPUWhenCreateRollbackFails(t *testing.T) { + m, id := newStartRollbackVGPUManager(t, func(context.Context, devices.VGPUAssignment) error { + return nil + }) + device := devices.VGPUDevice{ + Framework: devices.VGPUFrameworkVendorVFIO, + VFAddress: "0000:82:00.4", + ProfileType: "1148", + ProfileName: "NVIDIA L40S-2Q", + SysfsPath: "/sys/bus/pci/devices/0000:82:00.4", + } + cause := errors.New("create verification and rollback failed") + m.createVGPU = func(context.Context, string, string) (*devices.VGPUDevice, error) { + return nil, &devices.VGPUCreateCleanupPendingError{Device: device, Err: cause} + } + + _, err := m.startInstance(context.Background(), id, StartInstanceRequest{}) + require.ErrorIs(t, err, cause) + + stored, err := m.loadMetadata(id) + require.NoError(t, err) + assert.Equal(t, device.Framework, stored.GPUFramework) + assert.Equal(t, device.SysfsPath, stored.GPUDevicePath) + assert.NotNil(t, stored.GPUAssignedAt) +} + func TestStartRollbackClearsVGPUAssignmentAfterSuccessfulDestroy(t *testing.T) { var destroyed []devices.VGPUAssignment m, id := newStartRollbackVGPUManager(t, func(_ context.Context, assignment devices.VGPUAssignment) error { From abf009bd424045f0bbc71c7073acd3ed9e71bb30 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Mon, 10 Aug 2026 19:52:26 +0000 Subject: [PATCH 27/76] Preserve vGPU lifecycle compatibility --- lib/instances/create.go | 6 --- lib/instances/start.go | 10 ++--- lib/instances/vgpu.go | 8 ---- lib/instances/vgpu_test.go | 76 ++++++++++++++++++++++++++++++++++---- 4 files changed, 71 insertions(+), 29 deletions(-) diff --git a/lib/instances/create.go b/lib/instances/create.go index c221f698a..3ffa322e6 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -106,11 +106,6 @@ func (m *manager) createInstance( if hvType == "" { hvType = m.defaultHypervisor } - if req.GPU != nil && req.GPU.Profile != "" { - if err := validateVGPUHypervisor(hvType); err != nil { - return nil, fmt.Errorf("%w: %w", ErrInvalidRequest, err) - } - } starter, starterErr := m.getVMStarter(hvType) if starterErr == nil { if err := m.validateCreateVMConfig(starter, req, hvType); err != nil { @@ -118,7 +113,6 @@ func (m *manager) createInstance( } } - // 2. Validate image exists and is ready; auto-pull if not found log.DebugContext(ctx, "validating image", "image", req.Image) imageCtx, imageSpanEnd := m.startLifecycleStep(ctx, "resolve_image", diff --git a/lib/instances/start.go b/lib/instances/start.go index 39418ad7d..f95967699 100644 --- a/lib/instances/start.go +++ b/lib/instances/start.go @@ -48,11 +48,6 @@ func (m *manager) startInstance( return nil, fmt.Errorf("%w: cannot start from state %s, must be Stopped", ErrInvalidState, inst.State) } - if stored.GPUProfile != "" { - if err := validateVGPUHypervisor(stored.HypervisorType); err != nil { - return nil, fmt.Errorf("%w: %w", ErrInvalidState, err) - } - } // Release any assignment retained by an earlier failed release and // persist the cleared fields immediately, so a failure later in start // cannot leave on-disk metadata pointing at a device that is already @@ -177,8 +172,9 @@ func (m *manager) startInstance( if err != nil { if pendingDevice, ok := vgpuDevicePendingCleanup(err); ok { assignedAt := m.nowUTC() - setStoredVGPUDevice(stored, pendingDevice, assignedAt) - if saveErr := m.saveMetadata(meta); saveErr != nil { + retentionMeta := rollbackMeta + setStoredVGPUDevice(&retentionMeta.StoredMetadata, pendingDevice, assignedAt) + if saveErr := m.saveMetadata(&retentionMeta); saveErr != nil { log.ErrorContext(ctx, "failed to retain vGPU assignment after create rollback failure", "instance_id", id, "error", saveErr) return nil, fmt.Errorf("create vGPU for profile %s: %w; retain assignment: %v", stored.GPUProfile, err, saveErr) } diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index a52cfa5d1..290c474bb 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -8,17 +8,9 @@ import ( "time" "github.com/kernel/hypeman/lib/devices" - "github.com/kernel/hypeman/lib/hypervisor" "github.com/kernel/hypeman/lib/logger" ) -func validateVGPUHypervisor(hvType hypervisor.Type) error { - if hvType != hypervisor.TypeQEMU { - return fmt.Errorf("vGPU is only supported with qemu, got %s", hvType) - } - return nil -} - // VGPUAssignmentStartupGracePeriod bounds how long an assignment without a // persisted hypervisor PID is treated as potentially live. const VGPUAssignmentStartupGracePeriod = 5 * time.Minute diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index 3cfd04430..1ff759363 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -12,18 +12,12 @@ import ( "github.com/kernel/hypeman/lib/devices" "github.com/kernel/hypeman/lib/hypervisor" + "github.com/kernel/hypeman/lib/network" "github.com/kernel/hypeman/lib/paths" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -func TestValidateVGPUHypervisor(t *testing.T) { - t.Parallel() - - assert.NoError(t, validateVGPUHypervisor(hypervisor.TypeQEMU)) - assert.EqualError(t, validateVGPUHypervisor(hypervisor.TypeCloudHypervisor), "vGPU is only supported with qemu, got cloud-hypervisor") -} - func TestCleanupFailedCreateRetainsVGPUAssignment(t *testing.T) { t.Parallel() @@ -158,6 +152,22 @@ func TestVGPUDevicePendingCleanup(t *testing.T) { assert.Nil(t, retainedVGPUFromCreateError("inst-1", assignedAt, cause)) } +type startRetentionNetworkManager struct { + network.Manager + config network.NetworkConfig + releaseCalls int +} + +func (m *startRetentionNetworkManager) CreateAllocation(context.Context, network.AllocateRequest) (*network.NetworkConfig, error) { + config := m.config + return &config, nil +} + +func (m *startRetentionNetworkManager) ReleaseAllocation(context.Context, *network.Allocation) error { + m.releaseCalls++ + return nil +} + func newStartRollbackVGPUManager(t *testing.T, destroy func(context.Context, devices.VGPUAssignment) error) (*manager, string) { t.Helper() m := &manager{ @@ -194,6 +204,27 @@ func TestStartRetainsVGPUWhenCreateRollbackFails(t *testing.T) { m, id := newStartRollbackVGPUManager(t, func(context.Context, devices.VGPUAssignment) error { return nil }) + networkManager := &startRetentionNetworkManager{config: network.NetworkConfig{ + IP: "192.0.2.20", + MAC: "02:00:00:00:00:20", + TAPDevice: "tap-new", + }} + m.networkManager = networkManager + + previousProgramStart := time.Now().Add(-time.Hour).UTC() + previousExitCode := 23 + meta, err := m.loadMetadata(id) + require.NoError(t, err) + meta.NetworkEnabled = true + meta.IP = "192.0.2.10" + meta.MAC = "02:00:00:00:00:10" + meta.Entrypoint = []string{"old-entrypoint"} + meta.Cmd = []string{"old-command"} + meta.ProgramStartedAt = &previousProgramStart + meta.ExitCode = &previousExitCode + meta.ExitMessage = "previous exit" + require.NoError(t, m.saveMetadata(meta)) + device := devices.VGPUDevice{ Framework: devices.VGPUFrameworkVendorVFIO, VFAddress: "0000:82:00.4", @@ -206,7 +237,10 @@ func TestStartRetainsVGPUWhenCreateRollbackFails(t *testing.T) { return nil, &devices.VGPUCreateCleanupPendingError{Device: device, Err: cause} } - _, err := m.startInstance(context.Background(), id, StartInstanceRequest{}) + _, err = m.startInstance(context.Background(), id, StartInstanceRequest{ + Entrypoint: []string{"new-entrypoint"}, + Cmd: []string{"new-command"}, + }) require.ErrorIs(t, err, cause) stored, err := m.loadMetadata(id) @@ -214,6 +248,32 @@ func TestStartRetainsVGPUWhenCreateRollbackFails(t *testing.T) { assert.Equal(t, device.Framework, stored.GPUFramework) assert.Equal(t, device.SysfsPath, stored.GPUDevicePath) assert.NotNil(t, stored.GPUAssignedAt) + assert.Equal(t, []string{"old-entrypoint"}, stored.Entrypoint) + assert.Equal(t, []string{"old-command"}, stored.Cmd) + assert.Equal(t, previousProgramStart, *stored.ProgramStartedAt) + assert.Equal(t, previousExitCode, *stored.ExitCode) + assert.Equal(t, "previous exit", stored.ExitMessage) + assert.Equal(t, "192.0.2.10", stored.IP) + assert.Equal(t, "02:00:00:00:00:10", stored.MAC) + assert.Equal(t, 1, networkManager.releaseCalls) +} + +func TestStartDoesNotRestrictVGPUHypervisor(t *testing.T) { + m, id := newStartRollbackVGPUManager(t, func(context.Context, devices.VGPUAssignment) error { + return nil + }) + meta, err := m.loadMetadata(id) + require.NoError(t, err) + meta.HypervisorType = hypervisor.TypeCloudHypervisor + require.NoError(t, m.saveMetadata(meta)) + + cause := errors.New("create failed") + m.createVGPU = func(context.Context, string, string) (*devices.VGPUDevice, error) { + return nil, cause + } + + _, err = m.startInstance(context.Background(), id, StartInstanceRequest{}) + assert.ErrorIs(t, err, cause) } func TestStartRollbackClearsVGPUAssignmentAfterSuccessfulDestroy(t *testing.T) { From e5e35e2f508f6063ef8f4ffa250da4df7de1b0ff Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Mon, 10 Aug 2026 21:11:08 +0000 Subject: [PATCH 28/76] Reconcile vGPU protection from raw metadata and restore GPUAssignedAt ListInstancesForReconcile hydrated every instance (socket stat, UFFD health, /vm.info per instance) before the API served and again on each grace retry, while the protected-set scan only reads stored metadata fields. List raw metadata fail-closed instead, matching the release claim scan, and drop the now-unused loadInstances parameterization. Snapshot restore preserved the source's vGPU assignment path fields but not GPUAssignedAt, so a retained assignment lost its crash-recovery grace timestamp across a restore. Carry the timestamp with the rest of the assignment. --- lib/instances/manager.go | 21 +++++++++++++++++++-- lib/instances/query.go | 11 +---------- lib/instances/query_test.go | 6 ++++++ lib/instances/snapshot.go | 1 + lib/instances/snapshot_test.go | 4 ++++ 5 files changed, 31 insertions(+), 12 deletions(-) diff --git a/lib/instances/manager.go b/lib/instances/manager.go index 458275bb5..1bb1e53ec 100644 --- a/lib/instances/manager.go +++ b/lib/instances/manager.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "os" + "path/filepath" "strings" "sync" "time" @@ -737,9 +738,25 @@ func (m *manager) DefaultHypervisor() hypervisor.Type { return m.defaultHypervisor } -// ListInstancesForReconcile returns every instance or an invalid metadata error. +// ListInstancesForReconcile returns every instance's stored metadata or an +// invalid metadata error. It does not derive state: reconcile protection only +// needs raw metadata fields, and hydration would query the hypervisor of +// every instance on the host before the API serves. func (m *manager) ListInstancesForReconcile(ctx context.Context) ([]Instance, error) { - return m.loadInstances(ctx, false) + files, err := m.listMetadataFilesWithStatErrors(true) + if err != nil { + return nil, err + } + result := make([]Instance, 0, len(files)) + for _, file := range files { + id := filepath.Base(filepath.Dir(file)) + meta, err := m.loadMetadata(id) + if err != nil { + return nil, fmt.Errorf("load metadata for instance %s: %w", id, err) + } + result = append(result, Instance{StoredMetadata: meta.StoredMetadata}) + } + return result, nil } // ListInstances returns instances, optionally filtered by the given criteria. diff --git a/lib/instances/query.go b/lib/instances/query.go index eea66ab13..0621460da 100644 --- a/lib/instances/query.go +++ b/lib/instances/query.go @@ -786,16 +786,12 @@ func parseSentinelTimestamp(line, sentinelPrefix string) (time.Time, bool) { // listInstances returns all instances, skipping metadata files that cannot be loaded. func (m *manager) listInstances(ctx context.Context) ([]Instance, error) { - return m.loadInstances(ctx, true) -} - -func (m *manager) loadInstances(ctx context.Context, skipInvalid bool) ([]Instance, error) { ctx, span := m.tracerOrDefault().Start(ctx, "instances.list_metadata") defer span.End() log := logger.FromContext(ctx) log.DebugContext(ctx, "listing all instances") - files, err := m.listMetadataFilesWithStatErrors(!skipInvalid) + files, err := m.listMetadataFiles() if err != nil { log.ErrorContext(ctx, "failed to list metadata files", "error", err) return nil, err @@ -813,11 +809,6 @@ func (m *manager) loadInstances(ctx context.Context, skipInvalid bool) ([]Instan ) meta, err := m.loadMetadata(id) if err != nil { - if !skipInvalid { - hydrateSpan.RecordError(err) - hydrateSpan.End() - return nil, fmt.Errorf("load metadata for instance %s: %w", id, err) - } // Skip instances with invalid metadata log.WarnContext(hydrateCtx, "skipping instance with invalid metadata", "instance_id", id, "error", err) hydrateSpan.End() diff --git a/lib/instances/query_test.go b/lib/instances/query_test.go index 41aba54e8..ab3db29c3 100644 --- a/lib/instances/query_test.go +++ b/lib/instances/query_test.go @@ -34,6 +34,12 @@ func TestListInstancesForReconcileFailsOnInvalidMetadata(t *testing.T) { _, err = m.ListInstancesForReconcile(context.Background()) require.Error(t, err) assert.ErrorContains(t, err, "load metadata for instance invalid") + + require.NoError(t, os.Remove(m.paths.InstanceMetadata("invalid"))) + listed, err = m.ListInstancesForReconcile(context.Background()) + require.NoError(t, err) + require.Len(t, listed, 1) + assert.Equal(t, "valid", listed[0].Id) } func TestParseExitSentinelLine(t *testing.T) { diff --git a/lib/instances/snapshot.go b/lib/instances/snapshot.go index 2d2676c72..48c51328a 100644 --- a/lib/instances/snapshot.go +++ b/lib/instances/snapshot.go @@ -312,6 +312,7 @@ func (m *manager) restoreSnapshot(ctx context.Context, id string, snapshotID str restored.GPUFramework = sourceMeta.GPUFramework restored.GPUDevicePath = sourceMeta.GPUDevicePath restored.GPUMdevUUID = sourceMeta.GPUMdevUUID + restored.GPUAssignedAt = sourceMeta.GPUAssignedAt restored.HypervisorType = targetHypervisor restored.HypervisorVersion = targetHypervisorVersion restored.SocketPath = m.paths.InstanceSocket(id, starter.SocketName()) diff --git a/lib/instances/snapshot_test.go b/lib/instances/snapshot_test.go index b23e364ee..a92763b28 100644 --- a/lib/instances/snapshot_test.go +++ b/lib/instances/snapshot_test.go @@ -113,6 +113,8 @@ func TestRestoreSnapshotKeepsCurrentVGPUAssignment(t *testing.T) { meta.GPUFramework = devices.VGPUFramework("future-framework") meta.GPUDevicePath = "/sys/bus/pci/devices/0000:82:00.4" meta.GPUMdevUUID = "retained-uuid" + assignedAt := time.Now().UTC().Truncate(time.Second) + meta.GPUAssignedAt = &assignedAt require.NoError(t, mgr.saveMetadata(meta)) _, err = mgr.RestoreSnapshot(ctx, sourceID, snapshot.Id, RestoreSnapshotRequest{ @@ -126,6 +128,8 @@ func TestRestoreSnapshotKeepsCurrentVGPUAssignment(t *testing.T) { assert.Equal(t, devices.VGPUFramework("future-framework"), restored.GPUFramework) assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", restored.GPUDevicePath) assert.Equal(t, "retained-uuid", restored.GPUMdevUUID) + require.NotNil(t, restored.GPUAssignedAt) + assert.True(t, assignedAt.Equal(*restored.GPUAssignedAt)) } func TestStoppedSnapshotLifecycleAndForkAfterSourceDeletion(t *testing.T) { From 081ead159dec992fd30e4e6ed934a9778b27dce8 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Tue, 11 Aug 2026 14:46:50 +0000 Subject: [PATCH 29/76] Surface pending vGPU cleanup from start as a typed error MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When start's vGPU create fails with a pending device-layer cleanup, the error was returned untyped, so the API mapped it to a generic internal_error. Create already wraps the same condition in VGPUCleanupPendingError and surfaces vgpu_cleanup_pending with retained/unretained guidance. Wrap start's pending-cleanup error the same way — Retained reflects whether the retention record was persisted — and map it in the StartInstance handler ahead of the errors.Is cases so the wrapped cause cannot hide the pending cleanup. --- cmd/api/api/instances.go | 19 +++++++++++ cmd/api/api/instances_test.go | 62 +++++++++++++++++++++++++++++++++++ lib/instances/start.go | 6 ++-- lib/instances/vgpu_test.go | 39 ++++++++++++++++++++++ 4 files changed, 124 insertions(+), 2 deletions(-) diff --git a/cmd/api/api/instances.go b/cmd/api/api/instances.go index 681be6ab1..6d1589182 100644 --- a/cmd/api/api/instances.go +++ b/cmd/api/api/instances.go @@ -853,7 +853,26 @@ func (s *ApiService) StartInstance(ctx context.Context, request oapi.StartInstan result, err := s.InstanceManager.StartInstance(ctx, inst.Id, startReq) if err != nil { + var vgpuPending *instances.VGPUCleanupPendingError switch { + // Checked first: it wraps the original start error, so a later + // errors.Is case would match the cause and hide the pending vGPU cleanup. + case errors.As(err, &vgpuPending): + log.ErrorContext(ctx, "failed to start instance", "error", err) + message := fmt.Sprintf("failed to start instance: %v; vGPU release failed during rollback and instance %s retains the assignment, delete it or retry start to release it", vgpuPending.Err, vgpuPending.InstanceID) + innerCode := "vgpu_retained_instance" + if !vgpuPending.Retained { + message = fmt.Sprintf("failed to start instance: %v; vGPU release failed during rollback and the retention record for instance %s could not be saved; the assignment is recovered on the next startup reconcile", vgpuPending.Err, vgpuPending.InstanceID) + innerCode = "vgpu_unretained_instance" + } + return oapi.StartInstance500JSONResponse{ + Code: "vgpu_cleanup_pending", + Message: message, + InnerError: &oapi.ErrorDetail{ + Code: lo.ToPtr(innerCode), + Message: lo.ToPtr(vgpuPending.InstanceID), + }, + }, nil case errors.Is(err, instances.ErrInvalidState): return oapi.StartInstance409JSONResponse{ Code: "invalid_state", diff --git a/cmd/api/api/instances_test.go b/cmd/api/api/instances_test.go index fdd444f8e..738cf547c 100644 --- a/cmd/api/api/instances_test.go +++ b/cmd/api/api/instances_test.go @@ -1082,6 +1082,68 @@ func TestRestoreInstance_ErrorMapping(t *testing.T) { } } +// A retained-assignment error must win over the mapping of the start error +// it wraps, or the response omits the pending vGPU cleanup the caller has to +// resolve. +func TestStartInstance_VGPUCleanupPendingBeatsWrappedErrorMapping(t *testing.T) { + t.Parallel() + + resolved := &instances.Instance{ + StoredMetadata: instances.StoredMetadata{Id: "inst-1", Name: "inst-1"}, + State: instances.StateStopped, + } + + t.Run("retained", func(t *testing.T) { + t.Parallel() + svc := newTestService(t) + svc.InstanceManager = &errActionInstanceManager{Manager: svc.InstanceManager, err: &instances.VGPUCleanupPendingError{ + InstanceID: "inst-1", + Retained: true, + Err: fmt.Errorf("create vGPU for profile p: %w", instances.ErrInsufficientResources), + }} + + resp, rerr := svc.StartInstance(mw.WithResolvedInstance(ctx(), resolved.Id, resolved), oapi.StartInstanceRequestObject{Id: resolved.Id}) + require.NoError(t, rerr) + + pending, ok := resp.(oapi.StartInstance500JSONResponse) + require.True(t, ok, "expected 500 vgpu_cleanup_pending, got %T", resp) + assert.EqualValues(t, "vgpu_cleanup_pending", pending.Code) + assert.Contains(t, pending.Message, "inst-1") + assert.Contains(t, pending.Message, instances.ErrInsufficientResources.Error(), + "the underlying start failure must survive the cleanup guidance") + assert.Contains(t, pending.Message, "delete it or retry start") + require.NotNil(t, pending.InnerError) + require.NotNil(t, pending.InnerError.Code) + assert.Equal(t, "vgpu_retained_instance", *pending.InnerError.Code) + require.NotNil(t, pending.InnerError.Message) + assert.Equal(t, "inst-1", *pending.InnerError.Message) + }) + + t.Run("unretained", func(t *testing.T) { + t.Parallel() + svc := newTestService(t) + svc.InstanceManager = &errActionInstanceManager{Manager: svc.InstanceManager, err: &instances.VGPUCleanupPendingError{ + InstanceID: "inst-1", + Err: fmt.Errorf("create vGPU for profile p: %w", instances.ErrInsufficientResources), + }} + + resp, rerr := svc.StartInstance(mw.WithResolvedInstance(ctx(), resolved.Id, resolved), oapi.StartInstanceRequestObject{Id: resolved.Id}) + require.NoError(t, rerr) + + pending, ok := resp.(oapi.StartInstance500JSONResponse) + require.True(t, ok, "expected 500 vgpu_cleanup_pending, got %T", resp) + assert.EqualValues(t, "vgpu_cleanup_pending", pending.Code) + assert.Contains(t, pending.Message, "retention record for instance inst-1 could not be saved") + assert.Contains(t, pending.Message, "startup reconcile") + assert.NotContains(t, pending.Message, "delete") + require.NotNil(t, pending.InnerError) + require.NotNil(t, pending.InnerError.Code) + assert.Equal(t, "vgpu_unretained_instance", *pending.InnerError.Code) + require.NotNil(t, pending.InnerError.Message) + assert.Equal(t, "inst-1", *pending.InnerError.Message) + }) +} + func TestInstanceActions_ImageNotFoundMapsTo404(t *testing.T) { t.Parallel() diff --git a/lib/instances/start.go b/lib/instances/start.go index f95967699..2b7712af7 100644 --- a/lib/instances/start.go +++ b/lib/instances/start.go @@ -170,16 +170,18 @@ func (m *manager) startInstance( log.InfoContext(ctx, "creating vGPU for start", "instance_id", id, "profile", stored.GPUProfile) device, err := m.createVGPUDevice(ctx, stored.GPUProfile, id) if err != nil { + log.ErrorContext(ctx, "failed to create vGPU", "instance_id", id, "profile", stored.GPUProfile, "error", err) if pendingDevice, ok := vgpuDevicePendingCleanup(err); ok { assignedAt := m.nowUTC() retentionMeta := rollbackMeta setStoredVGPUDevice(&retentionMeta.StoredMetadata, pendingDevice, assignedAt) + wrapped := fmt.Errorf("create vGPU for profile %s: %w", stored.GPUProfile, err) if saveErr := m.saveMetadata(&retentionMeta); saveErr != nil { log.ErrorContext(ctx, "failed to retain vGPU assignment after create rollback failure", "instance_id", id, "error", saveErr) - return nil, fmt.Errorf("create vGPU for profile %s: %w; retain assignment: %v", stored.GPUProfile, err, saveErr) + return nil, &VGPUCleanupPendingError{InstanceID: id, Err: fmt.Errorf("%w; retain assignment: %v", wrapped, saveErr)} } + return nil, &VGPUCleanupPendingError{InstanceID: id, Retained: true, Err: wrapped} } - log.ErrorContext(ctx, "failed to create vGPU", "instance_id", id, "profile", stored.GPUProfile, "error", err) return nil, fmt.Errorf("create vGPU for profile %s: %w", stored.GPUProfile, err) } assignedAt := m.nowUTC() diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index 1ff759363..d9e599d9b 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -242,6 +242,10 @@ func TestStartRetainsVGPUWhenCreateRollbackFails(t *testing.T) { Cmd: []string{"new-command"}, }) require.ErrorIs(t, err, cause) + var pending *VGPUCleanupPendingError + require.ErrorAs(t, err, &pending) + assert.Equal(t, id, pending.InstanceID) + assert.True(t, pending.Retained) stored, err := m.loadMetadata(id) require.NoError(t, err) @@ -258,6 +262,41 @@ func TestStartRetainsVGPUWhenCreateRollbackFails(t *testing.T) { assert.Equal(t, 1, networkManager.releaseCalls) } +func TestStartReportsUnretainedVGPUWhenRetentionSaveFails(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("root bypasses directory permissions") + } + + m, id := newStartRollbackVGPUManager(t, func(context.Context, devices.VGPUAssignment) error { + return nil + }) + device := devices.VGPUDevice{ + Framework: devices.VGPUFrameworkVendorVFIO, + VFAddress: "0000:82:00.4", + ProfileType: "1148", + ProfileName: "NVIDIA L40S-2Q", + SysfsPath: "/sys/bus/pci/devices/0000:82:00.4", + } + cause := errors.New("create verification and rollback failed") + m.createVGPU = func(context.Context, string, string) (*devices.VGPUDevice, error) { + instanceDir := filepath.Dir(m.paths.InstanceMetadata(id)) + require.NoError(t, os.Chmod(instanceDir, 0o555)) + t.Cleanup(func() { _ = os.Chmod(instanceDir, 0o755) }) + return nil, &devices.VGPUCreateCleanupPendingError{Device: device, Err: cause} + } + + _, err := m.startInstance(context.Background(), id, StartInstanceRequest{}) + require.ErrorIs(t, err, cause) + var pending *VGPUCleanupPendingError + require.ErrorAs(t, err, &pending) + assert.Equal(t, id, pending.InstanceID) + assert.False(t, pending.Retained) + + stored, err := m.loadMetadata(id) + require.NoError(t, err) + assert.Empty(t, stored.GPUDevicePath, "retention save failed, so no assignment should be recorded") +} + func TestStartDoesNotRestrictVGPUHypervisor(t *testing.T) { m, id := newStartRollbackVGPUManager(t, func(context.Context, devices.VGPUAssignment) error { return nil From 45fbedce7dea25c7a2b83a07095423d95eef337d Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Tue, 11 Aug 2026 15:41:14 +0000 Subject: [PATCH 30/76] Cover retained-stub delete recovery and flag reconcile inventory failures The vgpu_cleanup_pending guidance tells callers to delete the retained instance to retry a failed vGPU release, but no test exercised delete against the minimal GPU-fields-only stub cleanupFailedCreate writes. Add one. Losing the reconcile inventory disables vendor VFIO reconciliation host-wide while releases fail closed on the same inventory, so log it at error level instead of warn. Also document the wholesale-restore assumption in cleanupStartVGPU. --- cmd/api/main.go | 5 +++- lib/instances/lifecycle_noop_test.go | 37 ++++++++++++++++++++++++++++ lib/instances/vgpu.go | 4 +++ 3 files changed, 45 insertions(+), 1 deletion(-) diff --git a/cmd/api/main.go b/cmd/api/main.go index 982e15c3f..f38a6a7c3 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -218,7 +218,10 @@ func liveInstanceVGPUDevicePaths(ctx context.Context, instanceManager instances. func reconcileVGPUs(ctx context.Context, instanceManager instances.Manager, logger *slog.Logger) { protected, retryAfter, err := liveInstanceVGPUDevicePaths(ctx, instanceManager) if err != nil { - logger.Warn("failed to list instances for vGPU reconcile protection; reconciling mdev only", "error", err) + // Operator-actionable: vendor VFIO reconciliation stays disabled + // host-wide (and releases fail closed on the same inventory) until + // the unreadable instance metadata is repaired. + logger.Error("failed to list instances for vGPU reconcile protection; reconciling mdev only", "error", err) protected = nil retryAfter = 0 } diff --git a/lib/instances/lifecycle_noop_test.go b/lib/instances/lifecycle_noop_test.go index 15f632b3a..c74d352fd 100644 --- a/lib/instances/lifecycle_noop_test.go +++ b/lib/instances/lifecycle_noop_test.go @@ -196,6 +196,43 @@ func TestDeletePersistsVGPUReleaseBeforeTeardown(t *testing.T) { assert.Equal(t, restartpolicy.BlockedReasonManualStop, persisted.RestartStatus.BlockedReason) } +// A failed create whose vGPU release also failed retains a minimal +// GPU-fields-only stub, and the API tells the caller to delete it to retry +// the release. Exercise that recovery path against the exact stub shape +// cleanupFailedCreate writes. +func TestDeleteReleasesRetainedCreateStub(t *testing.T) { + p := paths.New(t.TempDir()) + var destroyed []devices.VGPUAssignment + m := &manager{ + paths: p, + instanceLocks: sync.Map{}, + bootMarkerScans: sync.Map{}, + now: time.Now, + lifecycleEvents: newLifecycleSubscribers(), + destroyVGPU: func(_ context.Context, assignment devices.VGPUAssignment) error { + destroyed = append(destroyed, assignment) + return nil + }, + } + const id = "retained-stub" + require.NoError(t, m.ensureDirectories(id)) + assignedAt := time.Now().UTC() + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: id, + GPUFramework: devices.VGPUFrameworkVendorVFIO, + GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", + GPUAssignedAt: &assignedAt, + }})) + + require.NoError(t, m.DeleteInstance(context.Background(), id)) + + require.Len(t, destroyed, 1) + assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", destroyed[0].DevicePath) + assert.Equal(t, id, destroyed[0].InstanceID) + _, err := m.loadMetadata(id) + require.Error(t, err, "retained stub must be fully deleted") +} + func TestDeleteDropsStaleVGPUClaimedByLiveInstance(t *testing.T) { now := time.Now().UTC() m, id := newLifecycleNoopManagerWithInstance(t, StateStopped, now) diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index 290c474bb..f25c7c63e 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -85,6 +85,10 @@ func clearStoredVGPUDevice(stored *StoredMetadata) { stored.GPUAssignedAt = nil } +// cleanupStartVGPU wholesale-restores the pre-start metadata snapshot. That +// is safe while the instance lock serializes start and no cleanup registered +// after the vGPU one persists metadata; a future cleanup that writes metadata +// must switch this to targeted field restores. func (m *manager) cleanupStartVGPU(ctx context.Context, instanceID string, device *devices.VGPUDevice, assignedAt time.Time, rollbackMeta metadata) { assignment := devices.VGPUAssignment{ Framework: device.Framework, From c162c87c264810012d4c1335096a223c41594f2a Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Tue, 11 Aug 2026 17:16:52 +0000 Subject: [PATCH 31/76] Reject vendor VFIO vGPUs on Cloud Hypervisor and improve wedge forensics Vendor VFIO vGPUs boot but are non-functional on Cloud Hypervisor (upstream cloud-hypervisor#7572), and the wedged VM then blocks the VF release until startup reconcile. Reject the combination at create and start after the rollback handler is registered, so the rejected device is released through the normal cleanup path. Hypervisor selection otherwise stays caller policy and mdev on Cloud Hypervisor keeps working. Retain identity fields (name, image, hypervisor, data dir) on the failed-create retention record so it lists as a recognizable, deletable instance instead of a nameless phantom; resource claims released by rollback stay dropped. Expose the assigned vGPU device_path in the instance API - on vendor VFIO hosts mdev_uuid is empty and the sysfs path is the identity an operator needs when a release wedges. --- cmd/api/api/instances.go | 3 + lib/instances/create.go | 29 ++- lib/instances/start.go | 6 + lib/instances/vgpu.go | 24 ++- lib/instances/vgpu_test.go | 39 +++- lib/oapi/oapi.go | 404 +++++++++++++++++++------------------ openapi.yaml | 6 +- 7 files changed, 298 insertions(+), 213 deletions(-) diff --git a/cmd/api/api/instances.go b/cmd/api/api/instances.go index 6d1589182..a277b5476 100644 --- a/cmd/api/api/instances.go +++ b/cmd/api/api/instances.go @@ -1307,6 +1307,9 @@ func instanceToOAPI(inst instances.Instance) oapi.Instance { if inst.GPUMdevUUID != "" { gpu.MdevUuid = lo.ToPtr(inst.GPUMdevUUID) } + if inst.GPUDevicePath != "" { + gpu.DevicePath = lo.ToPtr(inst.GPUDevicePath) + } oapiInst.Gpu = gpu } diff --git a/lib/instances/create.go b/lib/instances/create.go index 3ffa322e6..23fd534b1 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -357,6 +357,12 @@ func (m *manager) createInstance( } } }) + // Checked after the cleanup handler is registered so rejection + // releases the device through the normal rollback. + if err := validateVGPUHypervisorCompat(gpuDevice.Framework, hvType); err != nil { + log.ErrorContext(ctx, "unsupported vGPU hypervisor combination", "instance_id", id, "framework", gpuDevice.Framework, "hypervisor", hvType) + return nil, err + } } if len(req.Devices) > 0 && m.deviceManager != nil { @@ -669,12 +675,25 @@ func (m *manager) cleanupFailedCreate(ctx context.Context, id string, retainedVG log.ErrorContext(ctx, "failed to retain instance data after vGPU cleanup failure", "instance_id", id, "error", err) return retentionSurvives() } + // Retain identity fields so the instance lists as a recognizable, + // deletable record rather than a nameless phantom, but drop resource + // claims (network, volumes, devices) that rollback already released. retained := StoredMetadata{ - Id: id, - GPUFramework: retainedVGPU.GPUFramework, - GPUDevicePath: retainedVGPU.GPUDevicePath, - GPUMdevUUID: retainedVGPU.GPUMdevUUID, - GPUAssignedAt: retainedVGPU.GPUAssignedAt, + Id: id, + Name: retainedVGPU.Name, + Image: retainedVGPU.Image, + ResolvedImage: retainedVGPU.ResolvedImage, + Platform: retainedVGPU.Platform, + CreatedAt: retainedVGPU.CreatedAt, + HypervisorType: retainedVGPU.HypervisorType, + HypervisorVersion: retainedVGPU.HypervisorVersion, + SocketPath: retainedVGPU.SocketPath, + DataDir: retainedVGPU.DataDir, + GPUProfile: retainedVGPU.GPUProfile, + GPUFramework: retainedVGPU.GPUFramework, + GPUDevicePath: retainedVGPU.GPUDevicePath, + GPUMdevUUID: retainedVGPU.GPUMdevUUID, + GPUAssignedAt: retainedVGPU.GPUAssignedAt, } if err := m.saveMetadata(&metadata{StoredMetadata: retained}); err != nil { log.ErrorContext(ctx, "failed to retain vGPU assignment metadata after cleanup failure", "instance_id", id, "error", err) diff --git a/lib/instances/start.go b/lib/instances/start.go index 2b7712af7..b44a68d50 100644 --- a/lib/instances/start.go +++ b/lib/instances/start.go @@ -191,6 +191,12 @@ func (m *manager) startInstance( cu.Add(func() { m.cleanupStartVGPU(ctx, id, device, assignedAt, rollbackMeta) }) + // Checked after the cleanup handler is registered so rejection + // releases the device through the normal rollback. + if err := validateVGPUHypervisorCompat(device.Framework, stored.HypervisorType); err != nil { + log.ErrorContext(ctx, "unsupported vGPU hypervisor combination", "instance_id", id, "framework", device.Framework, "hypervisor", stored.HypervisorType) + return nil, err + } if err := m.saveMetadata(meta); err != nil { log.ErrorContext(ctx, "failed to save metadata after vGPU creation", "instance_id", id, "error", err) return nil, fmt.Errorf("save metadata after vGPU creation: %w", err) diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index f25c7c63e..8c6351742 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -8,6 +8,7 @@ import ( "time" "github.com/kernel/hypeman/lib/devices" + "github.com/kernel/hypeman/lib/hypervisor" "github.com/kernel/hypeman/lib/logger" ) @@ -63,6 +64,18 @@ func retainedVGPUFromCreateError(instanceID string, assignedAt time.Time, err er } } +// validateVGPUHypervisorCompat rejects the one proven-broken combination: +// vendor VFIO vGPUs boot but are non-functional on Cloud Hypervisor (upstream +// cloud-hypervisor#7572), and the wedged VM then blocks the VF release until +// startup reconcile. Hypervisor selection otherwise remains caller policy; +// mdev on Cloud Hypervisor keeps working. See lib/devices/GPU.md. +func validateVGPUHypervisorCompat(framework devices.VGPUFramework, hvType hypervisor.Type) error { + if framework == devices.VGPUFrameworkVendorVFIO && hvType == hypervisor.TypeCloudHypervisor { + return fmt.Errorf("%w: vendor VFIO vGPUs are not functional on cloud-hypervisor, use qemu", ErrInvalidRequest) + } + return nil +} + func (m *manager) destroyVGPUAssignment(ctx context.Context, assignment devices.VGPUAssignment) error { destroy := m.destroyVGPU if destroy == nil { @@ -85,10 +98,13 @@ func clearStoredVGPUDevice(stored *StoredMetadata) { stored.GPUAssignedAt = nil } -// cleanupStartVGPU wholesale-restores the pre-start metadata snapshot. That -// is safe while the instance lock serializes start and no cleanup registered -// after the vGPU one persists metadata; a future cleanup that writes metadata -// must switch this to targeted field restores. +// cleanupStartVGPU wholesale-restores the pre-start metadata snapshot. The +// cleanup stack is LIFO, so cleanups registered after this one run before it +// and this restore would clobber anything they persisted; it is safe only +// while no such cleanup writes metadata and the instance lock serializes +// start. The snapshot is also a shallow copy (Phases shares its map), so it +// must be persisted before any Phases.Record on the live struct. Violating +// either invariant requires switching to targeted field restores. func (m *manager) cleanupStartVGPU(ctx context.Context, instanceID string, device *devices.VGPUDevice, assignedAt time.Time, rollbackMeta metadata) { assignment := devices.VGPUAssignment{ Framework: device.Framework, diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index d9e599d9b..d466f1883 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -47,12 +47,16 @@ func TestCleanupFailedCreateRetainsVGPUAssignment(t *testing.T) { assert.Equal(t, stored.GPUDevicePath, retained.GPUDevicePath) assert.Equal(t, stored.GPUMdevUUID, retained.GPUMdevUUID) assert.Equal(t, stored.GPUAssignedAt, retained.GPUAssignedAt) - assert.Empty(t, retained.Name) - assert.Empty(t, retained.GPUProfile) + // Identity fields survive so the retained record lists as a + // recognizable, deletable instance instead of a nameless phantom. + assert.Equal(t, stored.Name, retained.Name) + assert.Equal(t, stored.GPUProfile, retained.GPUProfile) + assert.Equal(t, stored.HypervisorType, retained.HypervisorType) + assert.Equal(t, stored.DataDir, retained.DataDir) + // Resource claims released by rollback stay dropped. assert.False(t, retained.NetworkEnabled) assert.Empty(t, retained.IP) assert.Empty(t, retained.Volumes) - assert.Empty(t, retained.DataDir) } func TestCleanupFailedCreateDeletesDataWithoutRetainedVGPU(t *testing.T) { @@ -315,6 +319,35 @@ func TestStartDoesNotRestrictVGPUHypervisor(t *testing.T) { assert.ErrorIs(t, err, cause) } +func TestValidateVGPUHypervisorCompat(t *testing.T) { + t.Parallel() + + err := validateVGPUHypervisorCompat(devices.VGPUFrameworkVendorVFIO, hypervisor.TypeCloudHypervisor) + require.ErrorIs(t, err, ErrInvalidRequest) + assert.NoError(t, validateVGPUHypervisorCompat(devices.VGPUFrameworkVendorVFIO, hypervisor.TypeQEMU)) + assert.NoError(t, validateVGPUHypervisorCompat(devices.VGPUFrameworkMdev, hypervisor.TypeCloudHypervisor)) +} + +func TestStartRejectsVendorVFIOOnCloudHypervisor(t *testing.T) { + var destroyed []devices.VGPUAssignment + m, id := newStartRollbackVGPUManager(t, func(_ context.Context, assignment devices.VGPUAssignment) error { + destroyed = append(destroyed, assignment) + return nil + }) + meta, err := m.loadMetadata(id) + require.NoError(t, err) + meta.HypervisorType = hypervisor.TypeCloudHypervisor + require.NoError(t, m.saveMetadata(meta)) + + _, err = m.startInstance(context.Background(), id, StartInstanceRequest{}) + require.ErrorIs(t, err, ErrInvalidRequest) + + require.Len(t, destroyed, 1, "the rejected vGPU must be released by rollback") + stored, err := m.loadMetadata(id) + require.NoError(t, err) + assert.Empty(t, stored.GPUDevicePath, "no assignment may be persisted for a rejected combination") +} + func TestStartRollbackClearsVGPUAssignmentAfterSuccessfulDestroy(t *testing.T) { var destroyed []devices.VGPUAssignment m, id := newStartRollbackVGPUManager(t, func(_ context.Context, assignment devices.VGPUAssignment) error { diff --git a/lib/oapi/oapi.go b/lib/oapi/oapi.go index fa568c18b..27d459268 100644 --- a/lib/oapi/oapi.go +++ b/lib/oapi/oapi.go @@ -1354,7 +1354,10 @@ type InstanceHypervisor string // InstanceGPU GPU information attached to the instance type InstanceGPU struct { - // MdevUuid mdev device UUID + // DevicePath sysfs path of the assigned vGPU device + DevicePath *string `json:"device_path,omitempty"` + + // MdevUuid mdev device UUID (mdev hosts only) MdevUuid *string `json:"mdev_uuid,omitempty"` // Profile vGPU profile name @@ -19073,205 +19076,206 @@ var swaggerSpec = []string{ "1VhohilpTMSQ6dcCWKt/FOrJwZPBYDDodgpNblf/exCipjt1fvYtlrDJ+QW0P2ZRhQFGUOQM5Swmoqj/", "TQw51EPkbug4/UwQYfd5O/HKfl7KVeGDuQ5zuB2Y8OciuMJQG/RziA69hXK+f3sRvVXOkZNfbbaR/Wp0", "kwgGgiKeJ7HW+Mb6tjMGORJbM6QkynBn8y6V6J2pvVmdug09Vhz9nhOxQO9PTythD4JMNA9oN3HgEg37", - "wLMbbcPOGhvJ2tHcxL3s4d3eB8ZtXVLxJMQvjmjrexhdErSh0Iphq6I4r7Kvaa0ymEVCmdknTTQrJlgz", - "5cdkPsrzkFaiHzkclHfvTo4rlILxk+1ng2fPe8/G2096e/Fgu4e3d5/0dvbxYLIbPd3d3tldkXHSIm3t", - "9ploQdU0EDBchIePXJh6KHq4KUmgJgTYwOcrymJ+VblngpGofu82ynVd98sx7K2HEMx8SbBUxk7QwDJO", - "4TYlkW7bRH7b1MiiplHYovjk7WD7c80sMLgGZvxW5Mz4L00yf2GrT70B+5tVHefteCsMyGWYrFstv/P2", - "izY42H9+sP+5i+ayJNaNsU5O97i5TaFXDuy4lobhUgE9g42zBHas8GHM5zZro9PtFIkl8DfcurWg5eJx", - "q2yppgPbDbORVfy7IUv5pKIYQEiIAbuLD7RI4AR/KJ1Q5MJrWeMo4XmMPKOXwf4Ch9eJpyToZsD/ZG1h", - "BsvTZD1oZQLAo6FEA2WaEYOjTzdiU5oP0Et4Fx7h1OhPdhCmUIjv48LxwgSm6PPlujbazOohn1tFBr7R", - "Wg3S/4Jp62WwttHVTRgx6AD9yuGbQq1ivG5kNa+DPrP8et0gu2FxsR1EBXRmZboD9FMhxxWSoJX8NiSx", - "f44swyqRYTYr+fl2xzuaWsqd83LNux2zop1uxy0U5KQvZ6e/K6l+6fz5pBiK2CI4gbNcJuPmiiYWDxtm", - "QqWikbRZGnpzm+QLW8OIxCOjpTQFf5oMT6vJFB858eX9KdoAyMO/IGtB1v/aLAJFK3fdzvO950+e7jx/", - "0grYqBzgehn0CPKPlwe3ViCNsnxkjRBNUz86e2eMDJFR34sgk/enPo5EJrhmPXrmrkG/8+f95z6eU8zz", - "ceJ58Cz4m4GPhQ0LQpYVvKgh4PB3mszpZMJ+/xhd7vxd0HT7+oncGW834NSajsL2rRPfi79kDCbjnqlH", - "FIbcAYISshGV6g2RMAN0ThQC+ukhHIEeUaQNW5Jz2FV2xYOEtbe7u/vs6f5OK7qyo/MOzgisXYFL2Y7A", - "O2LwJtp4c36OtjyCM2068AaAEmdWxwyfM2SLCQ+qAml/e7AbopKGi7ukGtv2PG1c8vdWT7OTsosO2c+F", - "Drd0yoOrvbs7eLq3/2y/3TG2dtiRuF7NYVxukFkei3jv7/wGSJNvD88QZN5OcFQ1orhQrBuNSt1oVFCt", - "waCs32Bgz54+2d/b3dluB68Wiu6wwIGVA1vlXYFDFyCKwG4ElmKZ9XabbouQOGUI7A2JEkzTw8jlMtRu", - "H4OmPhLmtXIT2lwM1vS/dHG1+LaVFamwDZlMGCMacIFyVtTw6K/3fX4RF2Yz1zbXw3quHsp/YXr1LA6Q", - "qVV2i6XMBJlTnssv0BBXJjl1knAubvRtk8Lyhsg8UcbPSCV6f/od8BRNa0gqklV1KEuNK9CSbjm5G53n", - "ComEibxpsVrtRputXzXhbsOp7a5Crqhwg0aMslhzrpytj7I8wkmUQ9UaXOynnhWAbUHufZYlCxNEnySc", - "MxTNMANvhPCghdCMJ3E/GHKqn4wmwfAFfoUSbtCVLwnJbEEXMwj9mRZh6JygDb+UmSGlWoHR/dQwGVuy", - "o0qN+2m4UiKWoaywIudcrydW3AP+NZ9UTI4Jn0pQChWkB/TrePMZFibqHzNToGieGl0yENkcGGKNmYdu", - "VHOT8olVcK3IARndZiVxJLiUiCR0CsVw3p/WEoVXJJcV6cLrIyerg21BusZzGLjKDOxU6zpmofsxkDjz", - "OTck0DAk562ISXTGyRSzHEq8eIRsLd791nGHMy7VqACAuuFgpRpB3YZckBKWrkhvL+xB7p3gvehY222W", - "ywb43urrJaoKN9U0wGaeGlzR8Gp1CxoMkfEyBNZK1K0SxquO2XQTVLgS6J9KaJV6+GBoA5JLPLbkYb1t", - "tolGCausup8lbdWW4Xy1Nzhvi5+2Gi7tDKvZCZvwAMjGDVyUzhJtw0IzIlIKlUtQTBglsdMlC1+lNXVB", - "ZnYiCYpzYlfOyKcC2wXH5ngDUAZzNjLKpjVeX++wjXnYjGF1WQfo177YJq5IhjNX34oc1soEBkqEyxzW", - "VtGWVI7C7qzlhgWZ5gkWyCIfthmyXKQJZZdtWpeLdMwTGiH9Qd0BPeFJwq9G+pH8Aeay2Wp2+oNRUw2g", - "czM4m4BnNqTWbzmFH/QsN2vpv2CJ2TLfbwE0S5swrWBI9k80IRZG7x2j1x6hV3HP93YGTWnpDY1WEtKX", - "IRhvyrktyQZPfC4DSXwrpRxXvYjEFozeiD1ZLk0hlRa3kkM7dS7A23l0qhkan4cBcmT4dQ0BBI0JJNi4", - "qS1zjRZssc1UgjUccjlDf+fjqkG0bXxtoDLYBiuxKASZBAPpYUdXGqTNG0tr4u3uTcAegK3qicJHN8RQ", - "WFdDrQxkauInb5bKic2IXTLq5mhKi7UoleECLQqcANtre8CAeuG3QGAwwMFItYASqlC3ZuFVM5RozIUA", - "qGct4XDmZgP4Jlrm0WvtAKbQ2xlZIEFSTNmQUVYYSQG1jCBG5kR46ahcaCVrSuI++pun4gE4dpqphUVd", - "B+P5dxLxK1aMccj8QerGc6nbOWTGsihyqMpfvqSbBa1PEwqkB4MTTAkoVEjVDE0EkTN/7qHilFrGu+Ii", - "bqz6s0DuFSgmAz5WpPglYT4rK5oJqoamoZH5ajlczlSWhadW/0SVYq+oXsx1dX+5JCIsJBZTKl5pFbri", - "HRVPOTFoKwA9AkX87F+GxRdwIy3ARcrm/+qaLH86Kxqv/lZ7zQMQcXi+h8ZsGzTBRiZfphbsU/WkrQ1V", - "gXyzVbAxy74EtOFilV0llKok4FUkaXVPtkt5q0flu9FsSRJVe997tv/0ScuSMJ/lrDMwWV/aNTdPV7jk", - "GnbqtI3f59n+s+fPd/f2n+/cyMPiEjga9qcpicPfH7RBrpU+rMm//vHP96c1r88+BDsPbjQok8IRHlJD", - "Gkd1QO9P//WPf7pR3XpAIUazDMXd4LdvjNJJ/J10gQJVF147J9kK/f6wYiTABZtBG2QyIWAGHZl165WD", - "qeFttJOCcYYjqhYBRo6vTFh58UoNUrqNO6g62JDIa9q28KOac8l8XGZ3brjO0X8a33CNFp61riwl83GT", - "H/p1vVfjhS69Fn6MQ4sQA1kUNV82cBfzucKyEjmt/44gwcGlci2ntZg3VsPb1nMOIIrFFlDzQgFDsOg1", - "edJ+5G9/bTs9v2XFrFNf8Q8rzmHzEbyR1TdwIweMvtH6HNYaf7AX4O2+Go39mm8ri+pVCsSVt+7N+22R", - "prtckKC4wW7en5eZeJMP6+C7QI92DHbJy7a7FZJooCYv6SRgQOMJ6RWBejYjBcnceAT1mbd47oFUyeiS", - "TyZVUNn9ZhBywNeBrCrXC1ZKayZdRK6dzaKOYG3AdIadfTnsaBVg2NlOh52a2yqYp5ji65HtoAqiMliF", - "Cl7mmdcGKd0MxgmPLk05M6iS3UcDlBLMJMoZHP6aV217sNo71O1k3t4UGNzEhDgtsS0Y05jM8JxC6Qfr", - "U5lWAjHJNVUSAkahnQMUcwOrVKnlameoXzNZhAflpOHSwWxhG9YN6vc4cxGt5btg4JtABVn2kQjetagA", - "mmO/fn3aNQEMEHpoBlaJb3QTNSPQDLLoolbHoPw9HD88TsgIxl3HxU+X19FP/gbPqiCSKGmBsktyqBEB", - "injOVB0wP22nyFXzt5avpJxBsJ8N/wAANNu7IRAUkwhOpFw+i1VCvwVx1/IG7EqHEgd2QyQMhwJ8SWFf", - "8RvrEK4PwBgbvDLMph0/rtt4CUdScVu3qzjVI3IdERLXkTXDr7SNlbdfBmPlX2ELxlNUSLZvQ7zz8uz6", - "d5dJBWNtWm0/pp9x1gMYELelFrLDYPBZUJgqoVUwvj3siFEIxzT0QpvUZnK9eq1/JdcKgMjjPDHocmHS", - "tazKXkbrVvzWKYRNB5oLsrYO3h3UhzPx5reqEGdD1R+iSJx9604Kwy3tzjlR7t1zS0aNO1StqFJxabmA", - "f/dKNcbGkFIX2QsebaebNRLcm4WtIhb9tmUyJMMpGWWCTOj1CuIxLxjFuIofUh6kIoPBAHlupPga7T1F", - "0QwLWRs7o9OZShbVAJy9AGjRZ1VPFEQR5gyFbXa+3E334XK0m91Ov/WQcHzuYfAs1Q6xIuloFUD1Uelt", - "s9b5DC/AitPoJHy6uzcY7O4MboVQ7YZ1g+U6Kj+xtQar7TSl1HnfWUd/JUrVb6HIZl4uYHslKCRFF8sk", - "lSA4PYDEmwxHBCVkAmh0ReHw9Z7FeterB28FKov8UtC/2yi7b84HX61NU3Rlwb3dNDrOuVgF+/Gfr3GI", - "NrCZaAm7LpBzt9sbPHm7vXuw/+Rge/suUKWLRWrK9nj6cfvqabKDJ3vJs8XT37dnT6c76W5QD7ukpgRP", - "G1r9Rb/bGGVTXpJV0KAKS0Mbdg4ZEfXKxPWK3pIklJGeLDKk1qcpruAFxv++9vzfzM5vZrBSdjivTtIX", - "IbAqF6dCWQ8DdGUns9J3UZ/NyfHqWdwqA6k+kDC91YcC5NVuMFAKYrvzmRAIOWt5Db3zXmx9Ea3Milt3", - "FYU87HDSg7vcsOIh8q4hIHizXnWBL19yAdvplAuqZunq26J4rcDrhrjpj1LFVWClPjqZMihL7v9chMn5", - "SpT+uNPtJB/3qmfG/t4eYstC/RYEaLfalwpahJFB1fvVqwCvlIqHMJHsWlfXY/5hu7f9HOIQko97Pwx6", - "z6sRB12zWv7ybbu3K78O2qyhX2vP1Wjafn6jiGu3nqso6BcaqhRX3ssWBNjSeFkE2l0dLuG2ssHl46U9", - "rkHmNAqgnyvp2ctt5AtNMUnwIgQC7xlqZU179IkMjcmUMtnGbrs7KAy3++mw00eHFokbdNmy5H+leSj2", - "7tEJTVMSUy1jGtW/OYNhp6Utrq5L3KwIiPsqIK31w+La8/UQCesSrtZdk/3PyMf9LO23nca7Cr0D7GpO", - "RQWwLnixi+gEYVarBErZHCc0ton0kBgJ8WoHDhGtJFnLA2QpBzo7SRdNuUJlCn1Le1vOmu2CxfjJNdhb", - "V2BmGILY+SKAKAVSF13Fvk6OUSZ4nEdl/mgCgy4RP0Rew0JbIeSvD8m9S/sGJGZPuEDr7RtNBo129smm", - "/a7ZJjXBNm/19mD9Vt+JUaTbybN4PQ8zL7XjYDeCSF+Tghgw0VSXvSYJepP50IKjv/FXcFnnNbbkSItE", - "eeYcLJqmlikp4G4BF0MorveYJERfU8uNIJ7EZZYElSUXXc9St588mzW5OMEjtTyQXwjJtK4C+EfQX4rZ", - "IjgwV9+zuEs2Bg5WWxqHV8/UBbKrVR3c07WSWONW+SbcploFhsvXbN4GL+XSM38XYNq+aLaMgOIYfkVI", - "e9OMtW+/dGFvjfbjuzDLPaSQ9tq6Hmr4qA69t4Ahd/2XscBarKsS717IPR8ii7dWM27CfK1ngfpW58Pe", - "/xgrMxr1D7Z++Mv/3fvwn0Frc01vlkT0YjKBQKNLsuiZKj9aR+9XEU+hxIAWpqeWVAhOwYYEaOL2MPrj", - "3R8UTGPxK06XpgARWl6Jnu21E/rLfzTHN3nL+A745FqS/ewKHHdRqVRxdx1tpERMXSy5SyTb7A8Z5KZd", - "koVEXuEvK9I4Qv1OFp94EejowoiBfcLmF2hMoZKiHDKt1eIoIpnWJmwtGWrKgXPgPoLgxG/HFiBzid/W", - "IWniCQh6f7oEl/v63dsfX7/79Xj0+uzFr4cno19e/DeEeFz1TA9xT9Pe3v4TWwTcX8ntYCGKm9dT6KNT", - "G6ZvXf2THBRawOmSKM1VDkEh5DpKcknnzkGokttXTlhO1r19JYLPhNpVKglFJVhI6IROCPj14TqxQTVU", - "OmKkEqqnW+MGZWj5xjaEM+wAJ/WK34fqVuitCK92ubHVRX8ya8dCDQhp4LBDxiuUuQ9oL1QCXoWL/fBe", - "RhuQOeJKvLrE2c2bgaIeFg0GIw+/cCWfwfMvUW3z3crymnOe9LR601CSIGhNNmsRjJyHpkxGQqfJ6TAd", - "B2R4a9qd0ikO+BlC/oQvUhXTDWhtxtTS/jeWBwvnMRzX6zWYY2mWqlZfoGYkkKrXnOaQaql2VJb+rwbP", - "5MzmrlIvtq6aqJoytWWr14bwMmIOqOGrspXLU+bQEXvw0fok3JV6lTczbyTNe3Pq1IeagrNigc700lzN", - "iCDeRsAHJQ7+DZfM5uW0QGEx1f8yIsqYVZfUo6VScDdLtFFYftwSFNnGy+bw1XUOTvF10QO4UrBc8j/C", - "PMo6S9svfwRM+jeutiSduCZgGDXlLozAXqWiVWviqGp5M3yqWp63eT948CyvWsH9ms5WjTjLPiqkGaLH", - "v2GqfuIC1MFmzJM7B3KHyz8mAjDg6jDtrTDOaUriEc/V6vNvS9fbK7+oP1rWr3WqLwYijirpvE28wKFy", - "lGNYXmm9HCTKBVWLc71eNpgb0iBd0VhYSOgIfi47hkKdnz6B0XgSSBh5SRgRNIIyqPo8ppiBxoTen3rV", - "8ExhxCW8VhCBXh+dWHODg/wF9ZEqID0Xd3l4dtLpduZEGJW7M+jv9gdwmDPCcEY7B53d/nZ/0AGtagZT", - "3ILS9TZ/2uYbF4rrSWwloR/dS/pLgVOi4IvfAkgAEHdoXwcVBE89JTLDVFgtMksAocAQDNVfA66/u1AP", - "zK3cNcve2mYKacaQ/UKy13ZzP4CgDGcHprkzGFhgc2WvX8jdMQkDW3+30aNlv62kOrtEAZj7JTXPyZbF", - "0n/qdvYG2zca06qhwNkNdfyOYZvES0A737/hQtyq0xNm0vJskrUNh/JPHBCSf9Z++6D3TOZpisXCLZi/", - "WhmXTYIxkQi7d40epySKNKuAYjx99JoR8xxhhbCJXBY5gxrG7kNNodVTYNp2m1yAFP3I48UXW8JKH85G", - "8anKzvRx+bREz1+OdgoyXt5I+8ghbBuqvQcC+hEXBbgf7KTsDZ7ffadHnE0SGinUKwjYxiNTCSE/CeCF", - "O+whLtDvOVcYFeH8j+hIW5l1XJBbt7yKtv6g8SdzvBMSMoOfEZFiZpIjzDtrDv3ScTYuifI4r7zVHOFD", - "aQ+4qRwIj7moQJCrHlH/2qoLg8vX0V4AgcH2aaYXPyDh793DCbeTLWqwPuSRg8qXKJfkMR0n62Ibl0JI", - "UJZ7SdTXQvOD+7yybBGBP+EpeiwE/JIUEl65W0uXwlYmcmYU4KAE+KZMWLTffVcV/t6WT7woGfBr6Kah", - "nIUyflUcL/rIralR+tUCIJYEgXnGy9fKmR7e13LCdu7jhMGMC0/Rt2vq2zW16pQbanFTgIPpnfIWNogb", - "WSD+fPaHG1sfvtke2tseWlkeGLmy1oW/83Ef2YjUiMcEyRnPkxiNCTJ4Ry72RGHRn35EWEQzOicAagdF", - "2vJE0QwLiCxJUYwVNj70RsPESrNE0dyWbq7n4hDLBa7jWEgyAhy+URP+ZBmBSBkjMdKfWOi+Ek5wqW63", - "OftBA3vRYHk1oqsZl6TA82PKu80hvVka7Ria7Q/ZWwv0qhcQgqkdr5EkAbjaFfYfzhAeMvvB946FuEAw", - "idOSc2EBmIHUIFOabVlObdMjHcmIh7B23hKGmerJjER0QiM7rUuysPGcwQZb1V3SA3bjfH9aJGygnc0w", - "XhvAM4bBeY+LZ8hSUtV/wyAIOkryuHRyOQghLMY4SYKFOaYJH+NkZNbnkgR8gi/hDbsofn1/501iPCam", - "Vnu2UDPOzN/5OGcqN3+PBb+SRAw7m/0hg0QMu9Yk7pYCIrqCQm5pxvU5Ezw1fW6ZIW79cUkWn/pDdhin", - "lDmKgE9wIjki1/Ad1LcCzAzDvRrowZymsB/8KJeKpz7yqaM7M0yeqyxXNqNEEtUNoX4OmeLoD4ft+Gnr", - "j7LHT+AsJjjWdOK9YqYEsnXTqOUI69mP4NWAu53AAgw7+iI1YR5TgZkysJ0FOCWa+lu6UVRH0Id0s77C", - "EWYo45mpLAFENcOa5CptAFYDThKk4Ci5b7XgDjvZMB8LvZeOG3H3DFBa7RhRhk5/9A7TYO9Z+DxJEgkS", - "iij5r/PXvyK4lfUemNfKcC2T0sG0wIDiHFynjqe9wNEMGUcVFBMcdmg87BTu3HgTxppLGy7T64FP8Qc9", - "tB9MN10a/9Dv66aMu/IA/faHaeVAn6UsNTigw86nLvIeTKma5ePi2YfwgjbBl51XGAHaMNfcJnASTAFp", - "xrvxzRWJWYy4vQWSBcKo5EB+4MqYMiwWqxIJA0tvV5BPTCSjtxh/DCFycdg5GLrYxWGnO+wQNoffbIDj", - "sPMpvALWa9lcuQ7us8K5WRDRk8Fgcz0Stl3fgM+yhWPgC+uAjVpRUXZT76CFYf1z+Qf+rfXPwvWDme68", - "hCYyir8zvj9CB4QnsfuaaMAFURO7MYtI4sTu9Yae+3ce6M2KSJLcN4E+FHkW7rECqf9RkSNsVnmMVprv", - "H5jiBvd1qVTM9g9Dv4/Ofh6wnlvbOZm7UOdwnRLAoLGqNDIvIyzROYypd66V7xfwa9/+1+l+gKl4kfDp", - "xYFR3VHCpyihzOYDeIHKWjywawkfGRia4juLSuOKxG0YSeJf//gnDIqy6b/+8U+L7f6vf/wTjvuWgVeD", - "GtMXM4KFGhOsLg7QL4RkPZzQOXGTgSqwZE7EAu0OrM0fHiGv1L2V0uSQDdkbonLBvLwJU69N2gatq0DP", - "h7KcSAvjo1+kE1tMxsQ2Buw27iybpbzXE90NwCHCDLwJ6FvR0QBgyVFTaNtqop2wydTMuWI0rYdpLgXr", - "recvilwrQ709M8AbMhhY4tC5gwd20mjj/PzFZh+BtmWoAgoGge5QNmPViP43nrSeJxmOUmUosMqGN0U4", - "w2OaUGdybKh2Yo5giqMZZaSMLy6wxl0TB26kmsccnp0gGwjZhVeH7PX5FphYFYlULkjXcgJhEUbLcmjc", - "5rlAD8C/qILosJ59d8gmBEOe0MmxYQIeCHeRD1g0zADIA2JcqapUXusOmUGStcjF+uClPCYJfAT9T7Ei", - "V3jRRUWtW1cdJcFKK8Syq18eMoP1ategB1AlyBtmH/iZGVLPRfLanC1BJolWjSEC35T9hr43JlwgG+Hs", - "Vfl33ZkkSzMsvWgpjl6f6/lNQRPkxh4ILb0+d7ux2UWSoyihQA0RZkM2hUAgB97LWWVXi4SyGRZxL+L6", - "EvDBnC4Zv0pIPG3isUc+kd2hJFPpJ3Ccfq6T62MTLmbLE9CH2ADUrfbcHdt32rnubIt/Jt+dLQR5A+ed", - "seASw2/M6n5z5LVw5IXXzTn1Qp61Y4fAeHcRv6aLBwr4dbS3vObmibdkD2HRQxsO2ga8Ilygs6MThONY", - "ECk3/73tfXqmhkpL+U/fj5oVP0ToiR0LFxb0z9pbqgTyWNjBGztqhN286vV1/fttq1J8p/GmK+rwlFfe", - "3d8etU5vco2UQm9Ja99ukrXBtlRGHMoMltTSA9EoIYX4UpxTn4rWWZVNGG9x5awUlyx7Pjl2B/L+7Mu2", - "65zV74Z7YIrHNYb4gIywmmrtV81+TNT8rthFhza9wvz8dZHm4P6koPs2RYfI/DGpi3Ft2TQXNEAnjRfo", - "S6IMvMld6um2h8DEz4lwp9oMdGFmXUzLfIoMTgtMCCwxq3XfE/NKO9XXtPdn0nxheW4isdgl/yaitFB2", - "y7VapeCe2BLQd6ffQg83Um+/XNiKJbDAIoMVdezcTmBZ3cBywaLNb5ErX5yiTVxjqcQKN28SF5Zsg6ZU", - "6Fn3JdcdMr/euJbprF5LGZokdDqzToCYTiBWT/n1u2GUO/cwyqJOtsCK2BDFx5j3e6YX2XqB50Qo9Pro", - "xKy/f6Vu/QFBq+tVJce8Vt6u79686hEW8bhwnjTLpPbJF1aYDP1Xcnnv/9Q9wnxW6sSDJoHxM/bfBJMj", - "E//ep/x/7fyU0LHAYvG/dn7CSUYZ+V+7hwlWRKrNOyOWwX3ddPetwDxi4tP6C60uGrAmNgXI2DUCf/FW", - "S5nfvf+nEvvNpG8k+Bfr+k32byP7+8u1Uvy3W3GnCoDp44E8XAWxhVYbHn2DtLkHo6mlSA/SpuJFKkFt", - "ZlwqePT48pttUDktKM6/Nlpa/8sDufL6cKR7ctyFhYSK0lDRwqYP3pMvwI3j3oVb2+/9OwIO0zGd5jyX", - "fmZiilU0I9Jm7SakyoAfm9hdXs+NgvdXTKWD+7w67l2u/kb3dyTx1zfUMG/j0Fsn87u32sr89n0t8xtE", - "U5vZbMtudF1Jps2GQGuHadqWjCvQr8sB4KFxhXQR9E4rKqW6gECDOBiy/631j98UwemHH1wKZT4Y7DyB", - "3wmbf/jBZVGyU0cqhClBbQW9w1+PwYs6hUBZKLJXJmzXx2FqdgPpubIC/3YKUulIbq8hOSr8piG10pC8", - "5VqtIdm9uFsVqVqa5N51JEdvoQW3mOJ/Ti3pT+4eqWhwMp9MaEQJgwIvkJgul+IBjSb3zTNyy4RkZv2R", - "XjBRRRJprUYWXGuNhF7WlP6S0TrdRpx3jrBSJM0UmgockUmemMoISM5yFfMr5mDfYYKughAt5xO63l1T", - "I9dIOAktXP23raZbVPy6b1XX1dp+nFlgPLPFa61yWYo2zdrlwxLv3eqULa7a+9cqHzOJGfVteekyrSEE", - "yhiZAlZpblLmii9LBLQ+evv2lUuP0+qJcEWxFHeVsFyR0CHzK2H10YuyxJh5wbWg1QcS23RaSBq0taVi", - "guOEMgLxxESGMtmq9ese9Fh8eQk4XJyvlQR8z8fSllt9OAn4wVjBvciaJ5Uq1rw0SPh1+4rT4uRNODWP", - "il9ZBhRgPCFZbwvnivdswu3WjBsUtjAQ5VmCI8Ch1K8ZiDSLcWAwEf2mALhA8CQhwkDfZbly4taQFYOj", - "zCtIbyWzC938KGeKJhddE84D+CUSYbaw+E9DVunMynyQhww59jBCQTIz4lqlSj1oynMJb0HKsN8lwskV", - "Xsghs5nL5nOo6itIZFAik6SPfuYAGoHwFFPmMV5TLvE7OWQXNE7IyGI+XCAqkZxxoQgjMUr5nMhqvwSL", - "hBIBkzjCeuUkSvECwNcMDqVZH54RA3BWQZbg+t+YxRQK7+meiykfDBlGO4MBSglm0uaJSzyBC8e2gWAQ", - "lQF9jzDaGzy3X9X2DQCC3fJv6NMkBJnzCI+TBSKaigGpQm3CBqa2EKYpKKy3b0KFNPtV2DdthbPKxlLp", - "6jrGXZSzMhMebP05KxLX9XapXDCYp/UCEiqKa9CCf4xJhPV6Ml7tB2AXeRTlInRB6q32KrL+OwqO3vTO", - "YanCeeYJmAwiEsOeM65mcKY5HKXN7xuoqiSqP8dFEzwkXCCMPLouLRokyoE1bgBM4UVZXpC5csEXm9+7", - "s6OPr2UE7vgboMDHcj8BEfHJpHIA119N5gCvyu9YJuE/6zk9cnVlfRYXUzxlXCoaOWZYL0P/TSFsrRCu", - "XtkgNU+4uPRlqyr9/sTFZVsNzIKf0seliPkz/AodEXp4ADT98P4IsIYbZUUTzb0raXX6Kk4pCF1USRfo", - "zFHC2VSfotIqf+9uA1+r2zCgcfoyFcbZXUD8aCVkZH80pWn1ZGzhT3AxRLbVh+ZFuvd7cEb9yhWiaZaQ", - "lEDp2p4hNr3ZJRwUlPmn0gNFuhmv1KfKz102uqA08QddJw4BXbkN2wDpfXm7gkw14dP1oINF5w5hL4A6", - "OGTvpIEDvzCupwtU8GAt0BqIf3Q1o9EMEAhBb9XtG4BCnGUXBfjy5gF6CQfZx6CGzjcMsL+mNckTYoAF", - "52l6cbBcnPX96Sl8ZMAHTRnWiwPkCrIW94fUb/mIgnoWCZYK/WpxEjcKZRx29EJhrW8W89u0WIMlOPaQ", - "hXAHGbmyDdIJuvAgCC8a8LEcv33Fp/KrcRWVJQ3MXBRHVnUE2iQs7jQFedAk7PjZHgxCSNstkRDNMO4Y", - "CHFpMK/4tCinUCFlnGVtydcOE6h4nqYraBhteLBqUsU8V3+RKiZCwMeWupuIG23gyJbSwpeaUC2InjvY", - "m0B+wVAmg28eXCrNVDvdDmF52jn4zf5rnqadbseOx8NFv4FwvwZRst7gcsiN3hkPNvKbWH4TQMgqs/cQ", - "IWs3h1WnmyXyN+aFP7230NnsHpAMQT6oGXG/JhHUG2/V4MN4gWwJI3t+HyMD+EsUJVySioPn8YBnWUNX", - "TWZsNhS5Ne7p4cW5qzbUJoLl3H567r78CnTvdbEibszITffeg0aWR/CYE4Hl0mwmXNQRl9ZFk3z1hPTl", - "tmRpqm0o5Btt3tzK2IowtZ6wzCLsB7GpPodzxVOsaASVj6IZ59Ij+wIe2dQos8bjgjLBtGK0XJtBcKFJ", - "9cKaoS+sGnFgTWYI+49sH3343OYdhL9wj8ovfvKsAgXH7zrRH6oDQGl2QckEZTiXREt1eUpQtIg0VzSl", - "rgiOZijCmcoFgSp+BKWU0TRPfdxrvWNzDBgdF9vpRReNc4USLKaglZmHLtgm4mlKWEzAPjdkM4LnVKuU", - "AiVYERYtepJA9d85QVdcXCYcx2BiyGIMnh6oHiiIpkAAEU+JwjFWGASdC33iRyaJ6aIoCGzUekauS2qI", - "h0zk7HtT0UA3e+EGeoEIQHZTOSsKR0Y4JiwKQlmff91s7Mvbos+Jqk/0gSKDbsVLHzJUyLe5uuF8HVFE", - "jywWmwu7jW3Y/AqhVzarsNXsD0dG/55H2szVzfGBHEzFEq86xV+HZ6kguq/Gu/Tw7iMuUJyb7rxTCWT+", - "Z/UJFQzFD7aCzFKzjbd1DBUV8oplvhHP2/rD/XlyC1veV8IJu42KfVMtpnLSXwPLtat6K577QEZMa0vy", - "bXIPx4JdRNeDiU9ceFzusRhbLcM2R7Pg2z53UgKD9sXZN7ZdZ9s24OG2bNvZZpdc+h4jp6wHMaJhDm7N", - "uI2s2poO/k2zUWqz81jmg7PI0nNxb2zxpGCEhjVmeJFwHP8ZgoRX+I8iLoSBvwBAjccEv+pZDf30ALDN", - "lUXeui5b8/3p6WYTlxBqJY8Q6hFzCC8lR3+WxssG3NdzIgSNLUopOjo9tuG6VCKRsz56nVKFFEeXhGRl", - "RgtkFfb1/BwQyHJB+QriR7dDmBKLjFOm1o6ifPVuBvPpVmXo75lPWjzvb+7w1u5wsOw/PnYGXAZyNswE", - "VmumCqu1dUYpm3CRGrkMj3muW9c8SC+T3k+DVDChCZELqUhqohIneQLHDWpD2Pq/9juzy12IydUnx6TL", - "ZUSkVErKmRwymyuSEaH71p/r9r0Aq6BDQOGCv54ZJvl1BO/pwZh4NayaVg0gm6CuaOegs4WzbCvGCjcE", - "iNnhfcaQfoJoPCQX6ZgnNEIJZZcSbST00qgnaC5Rov/YXBnON4LvvnR149ufLL3SJ2zCg7XjDM0WxPyn", - "yuqybM05Jh8dW3tJ/MPi+A9sdJitra+fLAhOelCP2AH3oFzRhH40rE43QqWikUk5wsXavT8tmGp/yE6J", - "EvodDKltSWIQDUC73MoEj7aG+WCwG2UU0N92CQwOGF7z4xR6PDp7Z9JQScrFojtk+h/Q8NvDM+PdnWBr", - "TfAGagsno5Ot12sCnM9hmf6NIwTNBFeiFwQ3/JtL8OYYI41nSDYcUZ6tUpV49qcPYbUS3De7wuO0KwDI", - "UzGbjQLYy6FxhW0Ic57kqf6H+eNkHa6ZwtHsPbz61Ui7Zjhru3ETfBSH0s4pJqa25YM4PcyCPdaYVb1w", - "bgogxFSiAYO3wKH6M1L3lzff++v4Fbo77Yq6urFfzdm675vPjsEhbPjr8ViOuaE0NxPFV1ufrjBttj79", - "mPDoUlooFt9sqPU2wFfXP5Z42NZFCGICZIYiC2FkgLKI7A5ZzQBpEH8kwkgRkVKGky2Ys2kEkL2dFQvP", - "OYUE7QjyVHqSxoCZlAB8N8Df6dmAoco14Hl0pa2s5b/jOyMVR2MS8ZQ4tPPNkOr2N0zVT1xUocu/Fr74", - "1lt/gATEFOzta9Dam3v8LPT2U3wNodJxbh3KbkQbL3n5ozEFdRHszbCzO5DDThcNOzvpsKN34AiDCRUr", - "tI9SynJFZB8dG/sWpOA+GSBJIs5i6UDXnQVvdyCbEnINWTZkdz6B7+5T7LFUBUv5xnYSYg/6PaS/h6Qd", - "tOEfOHsm4y4cuhjxXBlzvz1X9q2YKDCPbN67r9Y7I990+zac/G/2+FZ4FOyyZpfe1hvOnuVyRppNbq9M", - "IaNcjQHM2xUXlTP0dz6WXcTIlbGGC6n6S3xPf31mOriPQgO6q5sUGbBz/1ZhoEWFgXKtwmCNJsBSX8mO", - "OgxiI7nOuFCA4mhz7Q0NgSYByBE8wgl6fXQyZJFmRQZaUJCUA3eyeOjmFj782zl6cfSmi46h0CX6OR9v", - "9tFrlixcuXHjoxkyI4kZ5hVhhsaGakkcup7N2IF67jJYXHfwQJWjzckIeFbcXrkg8W5nRnAMEskfnVfc", - "dBZAHX7zSh8gAP41Xxbb3lkpfHTeECUWvcOJImK52VObJ8UKzAx7STsIOiu4GeBL3aF0yGtln0Y2MNAY", - "uzudAFLGp29FH+6+QOr9eMlMnIgptzfOAWmUQZIBjhePK5ZJzlDBHEMs0L+ui7IJTVnClpetVDCgy6bI", - "76/I5L6Sd1Ww5f9dTxfM9NE6mrLKPmkiLsqtrPX0uuTgmYFDto6qCGc4omrRRThJ7B1lb4IiIqVXiL9j", - "QfBlzK9Yf8jeFIVebEIvOjp713WOWhRTeWlasL7YPno9J0Lm42JwCA6a8RrDmpN4yBRHEU6iPNHiBplM", - "SAS5uFC/RTb4couhdO7w7JSdBIvNeFHt+aOrcRemCdi9kizqFLdltnpLkCjBNG0GH7eCGgQcQqjBWDfK", - "GaJsktiQqkhwKZFtqkcSOqXjxAYIyT56OyNI4pQMWZZgxohAuTRR8XrovUwQKXOT4K0bAJBeQ1FdVAIL", - "ZoIrG5qQcC6kiSbQFP7+FElFshVk9sa0fApzviPZ1jRue3ogI3VtDM2mEPsK0htiKMUsuKajPHEBjPca", - "im4G9NBS4mM5+G8FnU6J0KcCGyZrwvHMsXbLaQ59JWO5sd7lefFWu3qXRateVqKXsbcSGG5UYm3HnZtF", - "/QU6v6SN2IH20c2yiH/RH7Xsu5qtGh6EffSZswyV7vx3rJJ57iUJtjVglRT+2MxJ3sgrR7WSaLseVqt1", - "Zu1dZrq2xs96MNisx4yWhSvps00K79dHCIP7RXm47yJrj5u2KmhXFd20IeV/PZr+V0GBdwOj/8AoJ7eA", - "0f+q8u4B5/zh8E+CB/Wh8ugrvmdXbPdPj4R/V+nzBg4f4Nia0ucN17PBqysVpff2nXZqkm3xzyTB23jH", - "G8jvbtm/af0tVAZvsda5oDXBkzRTCxfQZn2VZdCZpB9Jv8ERXMSt3p0r+BYhnV+OPBydNgZ0/jlr4z9I", - "zKgtHUglOjkOFJ1/ZBiD/pmrXCxb+tbpYRHN6Jw0G92rJ9guUSZIL+MZOFdis2B2PdxdprDoTz8i27zF", - "XLX/gtqTANVPYhRTQSKVLEwdUM0RTB/fSSS41gTgOReL5igRc0R+Ejw9tLNZcx/aM2WNYWWcYbroxVjh", - "3txxmxUmtM+I7nTxlJrhIcrQyx/RBrlWwlS4QBOt+SA6KZaUXEeExBJoctMf8PagwbJJP5LRdNxmlCtq", - "lby2tWBQlEvFU7f3J8doA2qfTQnTe6FF/QlIspngcxqTuDLGzpwnZlW3Gxb0pnZXLVQUheuccmEG9yAy", - "TJsLafqRZlW2UITEjCnDMLi1VUGqZ8ok8ev+MGUuAMfukRvFtyvMan4bTtnRlAh1OO0iKs4NxPPmt2vu", - "MV9zfjKUu9Mqt50Lz1ltvG6XH9UybekuCj8UuXP3a7Z+//Wk9FD5KLN5rOl8XiikTWbzr4sEB/d3P9y3", - "ufz9I04BfUmc8u2ZyqEB3WKIYF5BTHdM5iThWQr10OHdTreTi6Rz0JkplR1sbUHs94xLdbD3/Olu59OH", - "T/9/AAAA//9gew1xCvABAA==", + "wLMbbcPOGhvJ2tHcxL3s4d3eB8ZtXVLxJMQvjmjrexhdErSh0Iphq6I4r7Kvaa0ymEVCmdknTTQrJhhK", + "kRmVgZF+q3IhJxbf0YrS2MlDBm+6gH8onfpyIbfGudzKIrpl82+2AJvjGWBz7AWTp2MyH+V5SDXSjxwY", + "y7t3J8doA34BbFlIoawSMMZPtp8Nnj3vPRtvP+ntxYPtHt7efdLb2ceDyW70dHd7Z3dFIkyLbLrbJ8gF", + "NeZAHHMRtT5y0fOhoOam3IWabGLjsa8oi/lV5foLBsj6vdvg23XdL4fWtx5CMCEnwVIZ80UDJzuFS55E", + "um0TkG4zNotSS2FD55O3g+3Ptf7A4BruiLciZ8atajAGChdC6g3Y36zqOG/H8mFALvFl3Wr5nbdftMHB", + "/vOD/c9dNJe8sW6MdXK6x81tighzGMy17BCXoejZkZyBsmNlImPVt8kknW6nyHeBv0EYqMVSF49bJXE1", + "HdhumI2sulYakqdPKvoKRKoYDL74QEsqTh+Big5Fir4WgY4SnsfIs8UZSDLww514uotuBtxi1kRnIEZN", + "MobWcQDTGipHUKYZMfgfdSM20/oAvYR34RFOjVpnB2Hql/iuNxwvTLyMPl+ua6NkrR7yudWv4ButbCH9", + "L5i2XgZrsl3dhJHODtCvHL4ptD3G67Zf8zqoWcuv1+3EGxau2yFnQGdW1DxAPxXiZSGgWoF0QxL758gy", + "rBKwZrMCG2B3vKOppdw5LwW+2zEr2ul23EJBqvxy0vy7kuqXzp9PiqFAMoITOMtljnCuaGJhumEmVCoa", + "SZs8oje3SeyxpZVIPDLKU1NMqkk8tQpW8ZGTqt6fog1AYvwLsoZt/a/NIn61ctftPN97/uTpzvMnrfCW", + "ygGuF42PIC16eXBr5eQoy0fWNtI09aOzd8b2ERmrQhH78v7Uh7fIBNesR8/cNeh3/rz/3IeZink+TjzH", + "osWkM6i2sGFBJLWCFzXEQf5OkzmdTNjvH6PLnb8Lmm5fP5E74+0G+FzTUdjsduIHFyzZqMm4Z8okhZGA", + "gKCEbATLekMkzACdE4WAfnoIR6DeFNnMluQcpJZd8SBh7e3u7j57ur/Tiq7s6LyDMwIjXOBStiPwjhi8", + "iTbenJ+jLY/gTJsOUwIQzplVfcPnDNkax4OqQNrfHuyGqKTh4i6pxrY9TxuX/L1VH+2k7KJDUnahWi6d", + "8uBq7+4Onu7tP9tvd4yteXgkrldzGJeyZJbHAvH7O78B0uTbwzMECcETHFVtOy5C7EajUjcaFRSRMODv", + "NxjYs6dP9vd2d7bbob6Fgk4snmHlwFZ5V+DQBYgisBuBpVhmvd2m2yIkThkCe0OiBNP0MHIpFrXbx4C8", + "j4R5rdyENheD1cCXLq4W37YybhUmK5OgY0QDLlDOitIi/fUu2S/iWW3m2uZ6WM/VQ2k5TK+ehScyJdRu", + "sZSZIHPKc/kFGuLK5MxOEs7Fjb5tUljeEJknythsqETvT78DnqJpDUlFsqoOZalxBYjTLSd3o/NcIZEw", + "kTctVqvdaLP1qybcbTi13VWAGhVu0AidFmvOlbP1wZ9HOIlyKKaDi/3UswIMMIAEyLJkYWL7k4RzhqIZ", + "ZuAkER7iEZrxJO4HI2H1k9EkGFXBr1DCDejzJSGZrTNjBqE/0yIMnRO04VdYM6RUq3u6nxomYyuJVKlx", + "Pw0XcMQylKxWpMLr9cSKe3jE5pOKJTThUwlKoYKshX4dBj/DwiQjYGbqJs1To0sGAq4DQ6wx89CNam5S", + "PrEKrhU5INHcrCSOBJcSkYROoUbP+9Na/vKKnLcii3l9QGd1sC1I1zg0A1eZQcNqXV4tdD8G8nk+54YE", + "GoacwRWhks44mWKWQ+UZj5CtIb7fOhxyxqUaFbhUNxysVCMoJ5ELUqLlFVn3hT3IvRO8Fx1ru81y2bjj", + "W329RFXhppoG2MxTgysaXq1uQYMhMl5G5loJBlaii9WhpG4CVlfWH6ASWqUebBnagJwXjy15EHSbbYJk", + "wiqr7mdJW7XVQV/tDc7bwrqtRnE7w2p2wiY8gP1xA8+ps0TbaNWMiJRCQRUUE0ZJ7HTJwoVqTV2QMJ5I", + "guKc2JUz8qnAdsGxOd7gs2LORkbZtMbr6x22MQ+bMayuNgH92hfbhDvJcELtW5HDWpl4RYlwmVrbKgiU", + "ylHYnbXcsCDTPMECWUDGNkOWizSh7LJN63KRjnlCI6Q/qPvFJzxJ+NVIP5I/wFw2W81OfzBqKk10bgZn", + "8wLNhtT6Lafwg57lZi0rGSwxW+b7LXCMtokeC0aK/0QTYtH93jF67RF6FY59b2fQlC3f0GglT34ZGfKm", + "nNuSbPDE5zKQW7hSynFFlUhsMfKN2JPl0tR3aXErORBW5wK8nUenmjjyedAkR4Zf14BJ0JhA3o+b2jLX", + "aMEW20wlWFoilzP0dz6uGkTbhv0GCpZtsBIiQ5BJML4fdnSlQdq8sbQm3u7eBIMC2KqeKHx0Q2iHdaXd", + "yviqJn7yZqnK2YzYJaNujqbiWYsKHi7+o4AvsL22xzGo16MLxCsDSo1UC6jsCuV0Fl6RRYnGXAhAoNYS", + "DmduNgC7omUevdYO9wq9nZEFEiTFlA0ZZYWRFMDUCGJkToSXJcuFVrKmJO6jv3kqHmB2p5laWDB4MJ5/", + "JxG/YsUYh8wfpG48l7qdQ2YsiyLPVKVcpG4WtD5NKJC1DE4wJaB+IlUzNBFEzvy5h2pmahnviou4sRjR", + "ArlXoMYN+FiR4peE+aysaCaoGpqGRuar5Sg+U/AWnlr9E1Vq0KJ6jdnV/eWSiLCQWEypeKVV6Ip3VDzl", + "xIDAACIK1Ba0fxkWX6CgtMA8KZv/q2uy/OmsaLz6W+01D9fEwQwfGrNt0AQbmTSeWrBP1ZO2NlQF0uBW", + "odks+xLQhguhdgVaqpKAVyil1T3ZLhOvnizgRrMlSVTtfe/Z/tMnLSvVfJazzqB3fWnX3Dxd4ZJr2KnT", + "Nn6fZ/vPnj/f3dt/vnMjD4vLK2nYn6bcEn9/0Aa5VvqwJv/6xz/fn9a8PvsQgz240aBMZkl4SA3ZJdUB", + "vT/91z/+6UZ16wGFGM0yQniD374xSifxd9IFClRdeO2cZCv0+8OKkQAXbAZtkMmEgBl0ZNatVw6mBgPS", + "TgrGGY6oWgQYOb4y0e7FKzWk6zbuoOpgQyKvaduiomrOJfNxmXS64TpH/2l8wzVaeNa64JXMx01+6Nf1", + "Xo0XuvRa+DEOLUIMZFFrfdnAXcznCstKQLf+O4K8C5dhtpxtY95YjbpbT4WAKBZb180LBQyhtdfkSfuR", + "v/217fT8lhWzTn3FP6w4h81H8EZW38CNHDD6RutTa2v8wV6At/tqNPZL0a2s9VepW1feujfvt0X28HKd", + "hOIGu3l/XsLkTT6sYwIDPdox2CUv2+5WSKKBmrxcmIABjSekVwTq2UQZJHPjEdRn3sLMBzI4o0s+mVSx", + "bvebsdEB9geSvVwvWCmtmXQRuXY2izqwtsH4GXb25bCjVYBhZzsddmpuq2D6ZIqvR7aDKrbLYBVYeZn+", + "XhukdDMYJzy6NFXWoHh3Hw1QSjCTKGdw+Gtete3Bau9Qt5N5e1NAgxMT4rTEtmBMYzLDcwoVKaxPZVoJ", + "xCTXVEkIGIV2DlDMDdpTpcSsnaF+zSQ3HpSThksHs4VtWDeo3+PMRbSW74KBbwKFbdlHInjXghVojv36", + "9WnXBDBA6KEZWCW+0U3UjEAzyKKLWnmF8vdw/PA4ISMYdx2uP11eRz8nHTyrgkiipMXvLsmhRgQo4jlT", + "dRz/tJ0iV00rW76ScgbBfjb8A3DZbO+GQFBMIjiRcvksVgn9FsRdyxuwKx1KHNgNkTAcCvAlhX3Fb6xD", + "uD4AY2zwqkObdvy4buMlHEnFbTmx4lSPyHVESFwH/Ay/0jZW3n4ZjJV/hS1GUFG42b4N8c7Ls+vfXYIX", + "jLVptf2YfsZZD9BJ3JZaJBEDDWixaqqEVoEe9yAtRiF41dALbTKuyfXqtf6VXCvAR4/zxIDehUnXsip7", + "Ga1b8VtnNjYdaC7I2vJ8d1C2zsSb36pwnQ1Vf4jadfatO6lXt7Q750S5d88tGTXuULXQS8Wl5QL+3SvV", + "GBtDSl1kL3i0nW7WSHBvFraKWFDeljmaDKdklAkyodcriMe8YBTjKqxJeZCKDAaDL7qR4mu09xRFMyxk", + "beyMTmcqWVQDcPYCWEqfVdRREEWYMxS22flyN92Hy9Fudjv91kPC8bkHDbRU0sSKpKNVuNlHpbfNWucz", + "vAArTqOT8Onu3mCwuzO4FXC2G9YNluuo/MSWQKy205RS531nHf2VKFW/hSLJermu7pWgkKtdLJNUguD0", + "ABJvMhwRlJAJgOQVCa3rPYv1rlcP3gpUNou2oH+3UXbfnA++WjKn6MpijrtpdJxzsYpB5D9f4xBtYDPR", + "EqReIOdutzd48nZ792D/ycH29l2AXReL1JTt8fTj9tXTZAdP9pJni6e/b8+eTnfS3aAedklNZaA2tPqL", + "frcxyqa8JKtYRhWWhjbsHDIi6gWT64XGJUkoIz1ZZEitT1NcwQuM/33t+b+Znd/MYKXscF6dpC9CYFUu", + "ToWyHgZ/y05mpe+iPpuT49WzuFUGUn0gYXqrDwXIq91goELFduczkRly1vIaeue92PoiWpkVt+4qCnnY", + "4aQHd7lhxUPkXQNm8Ga96gJfvuQCttMpF1TN0tW3RfFaASMOcdMfpYqreE99dDJlUC3d/7kIk/OVKP1x", + "p9tJPu5Vz4z9vT3yl0UgLgjQbrUvFbQII4Ni/KtXAV4pFQ9hItm1rq7H/MN2b/s5xCEkH/d+GPSeVyMO", + "uma1/OXbdm9Xfh20WUO/BKArHbX9/EYR1249V1HQLzRUwK68ly02saXxsja1uzpcwm1lg8vHS3tcQ/Jp", + "FEA/V9Kzl9vIF5pikuBFCJveM9TKmvboExkakyllso3ddndQGG7302Gnjw4tQDjosooX/fjNQw16j05o", + "mpKYahnTqP7NGQw7LW1xdV3iZrVJ3FcBaa0fFteer4dIWJdwte6a7H9GPu5nab/tNN5V6B1gV3MqKmCI", + "wYtdRCcIs1qBUsrmOKGxTaSHxEiIVztwQG0lyVoeIEs50NlJumjKFSpT6Fva23LWbBcsxk+uwd66AjPD", + "EMTOFwFEKQDE6Cr2dXKMMsHjPCrzRxMYdIn4IfIaRNsKIX99SO5d2jcgMXvCBVpv32gyaLSzTzbtd802", + "qQm2eau3B+u3+k6MIt1OnsXreZh5qR0HuxFy+5oUxICJprrsNUnQm8yHFhz9jb+CyzqvsSVHWiTKM+dg", + "0TS1TEkBdwu4GEJxvcckIfqaWm4E8SQusySoLLnoepa6/eTZrMnFCR6p5YH8QkimdRXAP4L+UswWwYG5", + "sqPFXbIxcGjf0ji8eqZckV2t6uCerpXEGrfKN+E2lVAwXL5m8zZ4KZee+bvA+PZFs2UEFMfwK0Lam+YS", + "APZLF/bWaD++C7PcQwppr63roQbb6kCFC3R0138ZC6zFuirx7oXc8yGyeGs14yYo2noWqG91Puz9j7Ey", + "o1H/YOuHv/zfvQ//GbQ21/RmSUQvJhMINLoki54pPqR19H4ViBUqH2hhempJheAUbEgAcm4Poz/e/UHB", + "NBa/4nRpChCh5VUO2l47ob/8R3N8k7eM74BPriXZzy4MchcFVBV319FGSsTUxZK7RLLN/pBBbtolWUjk", + "1SOzIo0j1O9k8YkXgY4ujBjYJ2x+gcYUCjzKIdNaLY4ikmltwpa4oaZKOQfuIwhO/HZsXTSX+G0dkiae", + "gKD3p0sovq/fvf3x9btfj0evz178engy+uXFf0OIx1XP9BD3NO3t7T+xtcn9ldwO1se4eZmHPjq1YfrW", + "1T/JQaEFnC6J0lzlEBRCrqMkl3TuHIQquX1Bh+Vk3dsXSPhMBGClklBUgkWqTuiEgF8frhMbVEOlI0Yq", + "oai7NW5QhpZvbEM4ww5wUq8mf6icht6K8GqXG1td9CezdizUYKMGDjtkvEL1/YD2QiXgVbjYD+9ltAGZ", + "I67yrEuc3bwZVuth0WAw8vALFxgaPP8SRUDfraz6OedJT6s3DZUSgtZksxbByHloymQkdJqcDtNxQIa3", + "pt0pneKAnyHkT/gixTrdgNZmTC3tf2PVsnAew3G9jIQ5lmapamUPakYCqXrNaQ6plmobgHcBWdjkrlIv", + "tq6aqJoytWWL6obwMmIOYOarspXLU+bQEXvw0fok3JV6lTczbyTNe3Pq1IeagrNigc700lzNiCDeRsAH", + "JTz/DZfM5uW0QGExRQkzIsqYVZfUo6VScDdLtFFYftwSFNnGy+bw1eUXTvF10QO4UrBc8j/CPMryT9sv", + "fwSo/Deu5CWduCZgGDXlLgwMX6WiVWviqGp5M3yqWp63eT948CyvWsH9ms5WjTjLPiqkGaLHv2GqfuIC", + "1MFmzJM7x5eHyz8mAjDg6ujxraDXaUriEc/V6vNvK+rbK78oi1qW1XWqLwYijirpvE28wKFylGNYXmm9", + "HCTKBVWLc71eNpgb0iBdLVtYSOgIfi47hvqhnz6B0XgSSBh5SRgRNILqrPo8ppiBxoTen3pF+ky9xiW8", + "VhCBXh+dWHODg/wF9ZEqID0Xd3l4dtLpduZEGJW7M+jv9gdwmDPCcEY7B53d/nZ/0AGtagZT3IKK+jZ/", + "2uYbF4rrSWwloR/dS/pLgVOi4IvfAkgAEHdoXwcVBE89JTLDVFgtMksAocAQDNVfQ7kBd6EemFu5a5a9", + "tc0U0owh+4Vkr+3mfgBBGc4OTHNnMLDA5spev5C7YxIGtv5uo0fLfltJdXaJAuj7S2qeky2Lpf/U7ewN", + "tm80plVDgbMb6vgdwzaJl4B2vn/DhbhVpyfMpOXZJGsbDuWfOCAk/6z99kHvmczTFIuFWzB/tTIumwRj", + "IhF27xo9TkkUaVYBNYL66DUj5jnCCmETuSxyBqWV3YeaQqunwLTtNrkAKfqRx4svtoSVPpyN4lOVnenj", + "8mmJnr8c7RRkvLyR9pFD2DZUew8E9CMu6oI/2EnZGzy/+06POJskNFKoVxCwjUemEkJ+EsALd9hDXKDf", + "c64wKsL5H9GRtjLruCC3bnkVbf1B40/meCckZAY/IyLFzCRHmHfWHPql42xcEuVxXnmrOcI/Oe7Ym8qB", + "8JiLCgS56hH1r626MLh8He0FEBhsn2Z68QMS/t49nHA72aI07EMeOSjIiXJJHtNxsi62cSmEBGW5l0R9", + "LTQ/uM8ryxYR+BOeosdCwC9JIeGVu7V0KWxlImdGAQ5KgG/KhEX73XdV4e9t+cSLkgG/hm4aylko41fF", + "8aKP3JoapV8tAGJJEJhnvHytnOnhfS0nbOc+ThjMuPAUfbumvl1Tq065oRY3BTiY3ilvYYO4kQXiz2d/", + "uLH14Zvtob3toZXlgZEra134Ox/3kY1IjXhMkJzxPInRmCCDd+RiTxQW/elHhEU0o3MCoHZQpC1PFM2w", + "gMiSFMVYYeNDbzRMrDRLFM1t6eZ6Lg6xXOA6joUkI8DhGzXhT5YRiJQxEiP9iYXuK+EEl8qJm7MfNLAX", + "DZZXI7qacUkKPD+mvNsc0pul0Y6h2f6QvbVAr3oBIZja8RpJEoCrXWH/4QzhIbMffO9YiAsEkzgtORcW", + "gBlIDTKl2Zbl1DY90pGMeAhr5y1hmKmezEhEJzSy07okCxvPGWywVd0lPWA3zvenRcIG2tkM47UBPGMY", + "nPe4eIYsJVX9NwyCoKMkj0snl4MQwmKMkyRYmGOa8DFORmZ9LknAJ/gS3rCLUjpcSm8S4zExJeSzhZpx", + "Zv7OxzlTufl7LPiVJGLY2ewPGSRi2LUmcbcUENEVFHJLM67PmeCp6XPLDHHrj0uy+NQfssM4pcxRBHyC", + "E8kRuYbvoL4VYGYY7tVAD+Y0hf3gR7lUPPWRTx3dmWHyXGW5shklkqhuCPVzyBRHfzhsx09bf5Q9fgJn", + "McGxphPvFTMlkK2bRi1HWM9+BK8G3O0EFmDY0RepCfOYCsyUge0swCnR1N/SjaI6AlRMra9whBnKeGYq", + "SwBRzbAmuUobgNWAkwQpOEruWy24w042zMdC76XjRtw9A5RWO0aUodMfvcM02HsWPk+SRIKEIkr+6/z1", + "rwhuZb0H5rUyXMukdDAtMKA4B9ep42kvcDRDxlEFxQSHHRoPO4U7N96EsebShsv0euBT/EEP7QfTTZfG", + "P/T7uinjrjxAv/1hWjnQZylLDQ7osPOpi7wHU6pm+bh49iG8oE3wZecVRoA2zDW3CZwEU0Ca8W58c0Vi", + "FiNub4FkgTAqOZAfuDKmDIvFqkTCwNLbFeQTE8noLcYfQ4hcHHYOhi52cdjpDjuEzeE3G+A47HwKr4D1", + "WjZXroP7rHBuFkT0ZDDYXI+Ebdc34LNs4Rj4wjpgo1ZUlN3UO2hhWP9c/oF/a/2zcP1gpjsvoYmM4u+M", + "74/QAeFJ7L4mGnBB1MRuzCKSOLF7vaHn/p0HerMikiT3TaAPRZ6Fe6xA6n9U5AibVR6jleb7B6a4wX1d", + "KhWz/cPQ76Oznwes59Z2TuYu1DlcpwQwaKwqjczLCEt0DmPqnWvl+wX82rf/dbofYCpeJHx6cWBUd5Tw", + "KUoos/kAXqCyFg/sWsJHBoam+M6i0rgicRtGkvjXP/4Jg6Js+q9//NNiu//rH/+E475l4NWgxvTFjGCh", + "xgSriwP0CyFZDyd0TtxkoAosmROxQLsDa/OHR8grdW+lNDlkQ/aGqFwwL2/C1GuTtkHrKtDzoSwn0sL4", + "6BfpxBaTMbGNAbuNO8tmKe/1RHcDcIgwA28C+lZ0NABYctQU2raaaCdsMjVzrhhN62GaS8F66/mLItfK", + "UG/PDPCGDAaWOHTu4IGdNNo4P3+x2UegbRmqgIJBoDuUzVg1ov+NJ63nSYajVBkKrLLhTRHO8Jgm1Jkc", + "G6qdmCOY4mhGGSnjiwuscdfEgRup5jGHZyfIBkJ24dUhe32+BSZWRSKVC9K1nEBYhNGyHBq3eS7QA/Av", + "qiA6rGffHbIJwZAndHJsmIAHwl3kAxYNMwDygBhXqiqV17pDZpBkLXKxPngpj0kCH0H/U6zIFV50UVHr", + "1lVHSbDSCrHs6peHzGC92jXoAVQJ8obZB35mhtRzkbw2Z0uQSaJVY4jAN2W/oe+NCRfIRjh7Vf5ddybJ", + "0gxLL1qKo9fnen5T0AS5sQdCS6/P3W5sdpHkKEooUEOE2ZBNIRDIgfdyVtnVIqFshkXci7i+BHwwp0vG", + "rxIST5t47JFPZHcoyVT6CRynn+vk+tiEi9nyBPQhNgB1qz13x/addq472+KfyXdnC0HewHlnLLjE8Buz", + "ut8ceS0ceeF1c069kGft2CEw3l3Er+nigQJ+He0tr7l54i3ZQ1j00IaDtgGvCBfo7OgE4TgWRMrNf297", + "n56podJS/tP3o2bFDxF6YsfChQX9s/aWKoE8Fnbwxo4aYTeven1d/37bqhTfabzpijo85ZV397dHrdOb", + "XCOl0FvS2rebZG2wLZURhzKDJbX0QDRKSCG+FOfUp6J1VmUTxltcOSvFJcueT47dgbw/+7LtOmf1u+Ee", + "mOJxjSE+ICOsplr7VbMfEzW/K3bRoU2vMD9/XaQ5uD8p6L5N0SEyf0zqYlxbNs0FDdBJ4wX6kigDb3KX", + "errtITDxcyLcqTYDXZhZF9MynyKD0wITAkvMat33xLzSTvU17f2ZNF9YnptILHbJv4koLZTdcq1WKbgn", + "tgT03em30MON1NsvF7ZiCSywyGBFHTu3E1hWN7BcsGjzW+TKF6doE9dYKrHCzZvEhSXboCkVetZ9yXWH", + "zK83rmU6q9dShiYJnc6sEyCmE4jVU379bhjlzj2MsqiTLbAiNkTxMeb9nulFtl7gOREKvT46MevvX6lb", + "f0DQ6npVyTGvlbfruzeveoRFPC6cJ80yqX3yhRUmQ/+VXN77P3WPMJ+VOvGgSWD8jP03weTIxL/3Kf9f", + "Oz8ldCywWPyvnZ9wklFG/tfuYYIVkWrzzohlcF833X0rMI+Y+LT+QquLBqyJTQEydo3AX7zVUuZ37/+p", + "xH4z6RsJ/sW6fpP928j+/nKtFP/tVtypAmD6eCAPV0FsodWGR98gbe7BaGop0oO0qXiRSlCbGZcKHj2+", + "/GYbVE4LivOvjZbW//JArrw+HOmeHHdhIaGiNFS0sOmD9+QLcOO4d+HW9nv/joDDdEynOc+ln5mYYhXN", + "iLRZuwmpMuDHJnaX13Oj4P0VU+ngPq+Oe5erv9H9HUn89Q01zNs49NbJ/O6ttjK/fV/L/AbR1GY227Ib", + "XVeSabMh0NphmrYl4wr063IAeGhcIV0EvdOKSqkuINAgDobsf2v94zdFcPrhB5dCmQ8GO0/gd8LmH35w", + "WZTs1JEKYUpQW0Hv8Ndj8KJOIVAWiuyVCdv1cZia3UB6rqzAv52CVDqS22tIjgq/aUitNCRvuVZrSHYv", + "7lZFqpYmuXcdydFbaMEtpvifU0v6k7tHKhqczCcTGlHCoMALJKbLpXhAo8l984zcMiGZWX+kF0xUkURa", + "q5EF11ojoZc1pb9ktE63EeedI6wUSTOFpgJHZJInpjICkrNcxfyKOdh3mKCrIETL+YSud9fUyDUSTkIL", + "V/9tq+kWFb/uW9V1tbYfZxYYz2zxWqtclqJNs3b5sMR7tzpli6v2/rXKx0xiRn1bXrpMawiBMkamgFWa", + "m5S54ssSAa2P3r595dLjtHoiXFEsxV0lLFckdMj8Slh99KIsMWZecC1o9YHENp0WkgZtbamY4DihjEA8", + "MZGhTLZq/boHPRZfXgIOF+drJQHf87G05VYfTgJ+MFZwL7LmSaWKNS8NEn7dvuK0OHkTTs2j4leWAQUY", + "T0jW28K54j2bcLs14waFLQxEeZbgCHAo9WsGIs1iHBhMRL8pAC4QPEmIMNB3Wa6cuDVkxeAo8wrSW8ns", + "Qjc/ypmiyUXXhPMAfolEmC0s/tOQVTqzMh/kIUOOPYxQkMyMuFapUg+a8lzCW5Ay7HeJcHKFF3LIbOay", + "+Ryq+goSGZTIJOmjnzmARiA8xZR5jNeUS/xODtkFjRMyspgPF4hKJGdcKMJIjFI+J7LaL8EioUTAJI6w", + "XjmJUrwA8DWDQ2nWh2fEAJxVkCW4/jdmMYXCe7rnYsoHQ4bRzmCAUoKZtHniEk/gwrFtIBhEZUDfI4z2", + "Bs/tV7V9A4Bgt/wb+jQJQeY8wuNkgYimYkCqUJuwgakthGkKCuvtm1AhzX4V9k1b4ayysVS6uo5xF+Ws", + "zIQHW3/OisR1vV0qFwzmab2AhIriGrTgH2MSYb2ejFf7AdhFHkW5CF2Qequ9iqz/joKjN71zWKpwnnkC", + "JoOIxLDnjKsZnGkOR2nz+waqKonqz3HRBA8JFwgjj65LiwaJcmCNGwBTeFGWF2SuXPDF5vfu7OjjaxmB", + "O/4GKPCx3E9ARHwyqRzA9VeTOcCr8juWSfjPek6PXF1Zn8XFFE8Zl4pGjhnWy9B/UwhbK4SrVzZIzRMu", + "Ln3Zqkq/P3Fx2VYDs+Cn9HEpYv4Mv0JHhB4eAE0/vD8CrOFGWdFEc+9KWp2+ilMKQhdV0gU6c5RwNtWn", + "qLTK37vbwNfqNgxonL5MhXF2FxA/WgkZ2R9NaVo9GVv4E1wMkW31oXmR7v0enFG/coVomiUkJVC6tmeI", + "TW92CQcFZf6p9ECRbsYr9anyc5eNLihN/EHXiUNAV27DNkB6X96uIFNN+HQ96GDRuUPYC6AODtk7aeDA", + "L4zr6QIVPFgLtAbiH13NaDQDBELQW3X7BqAQZ9lFAb68eYBewkH2Maih8w0D7K9pTfKEGGDBeZpeHCwX", + "Z31/egofGfBBU4b14gC5gqzF/SH1Wz6ioJ5FgqVCv1qcxI1CGYcdvVBY65vF/DYt1mAJjj1kIdxBRq5s", + "g3SCLjwIwosGfCzHb1/xqfxqXEVlSQMzF8WRVR2BNgmLO01BHjQJO362B4MQ0nZLJEQzjDsGQlwazCs+", + "LcopVEgZZ1lb8rXDBCqep+kKGkYbHqyaVDHP1V+kiokQ8LGl7ibiRhs4sqW08KUmVAui5w72JpBfMJTJ", + "4JsHl0oz1U63Q1iedg5+s/+ap2mn27Hj8XDRbyDcr0GUrDe4HHKjd8aDjfwmlt8EELLK7D1EyNrNYdXp", + "Zon8jXnhT+8tdDa7ByRDkA9qRtyvSQT1xls1+DBeIFvCyJ7fx8gA/hJFCZek4uB5POBZ1tBVkxmbDUVu", + "jXt6eHHuqg21iWA5t5+euy+/At17XayIGzNy0733oJHlETzmRGC5NJsJF3XEpXXRJF89IX25LVmaahsK", + "+UabN7cytiJMrScsswj7QWyqz+Fc8RQrGkHlo2jGufTIvoBHNjXKrPG4oEwwrRgt12YQXGhSvbBm6Aur", + "RhxYkxnC/iPbRx8+t3kH4S/co/KLnzyrQMHxu070h+oAUJpdUDJBGc4l0VJdnhIULSLNFU2pK4KjGYpw", + "pnJBoIofQSllNM1TH/da79gcA0bHxXZ60UXjXKEEiyloZeahC7aJeJoSFhOwzw3ZjOA51SqlQAlWhEWL", + "niRQ/XdO0BUXlwnHMZgYshiDpweqBwqiKRBAxFOicIwVBkHnQp/4kUliuigKAhu1npHrkhriIRM5+95U", + "NNDNXriBXiACkN1UzorCkRGOCYuCUNbnXzcb+/K26HOi6hN9oMigW/HShwwV8m2ubjhfRxTRI4vF5sJu", + "Yxs2v0Lolc0qbDX7w5HRv+eRNnN1c3wgB1OxxKtO8dfhWSqI7qvxLj28+4gLFOemO+9UApn/WX1CBUPx", + "g60gs9Rs420dQ0WFvGKZb8Tztv5wf57cwpb3lXDCbqNi31SLqZz018By7areiuc+kBHT2pJ8m9zDsWAX", + "0fVg4hMXHpd7LMZWy7DN0Sz4ts+dlMCgfXH2jW3X2bYNeLgt23a22SWXvsfIKetBjGiYg1szbiOrtqaD", + "f9NslNrsPJb54Cyy9FzcG1s8KRihYY0ZXiQcx3+GIOEV/qOIC2HgLwBQ4zHBr3pWQz89AGxzZZG3rsvW", + "fH96utnEJYRaySOEesQcwkvJ0Z+l8bIB9/WcCEFji1KKjk6PbbgulUjkrI9ep1QhxdElIVmZ0QJZhX09", + "PwcEslxQvoL40e0QpsQi45SptaMoX72bwXy6VRn6e+aTFs/7mzu8tTscLPuPj50Bl4GcDTOB1Zqpwmpt", + "nVHKJlykRi7DY57r1jUP0suk99MgFUxoQuRCKpKaqMRJnsBxg9oQtv6v/c7schdicvXJMelyGREplZJy", + "JofM5opkROi+9ee6fS/AKugQULjgr2eGSX4dwXt6MCZeDaumVQPIJqgr2jnobOEs24qxwg0BYnZ4nzGk", + "nyAaD8lFOuYJjVBC2aVEGwm9NOoJmkuU6D82V4bzjeC7L13d+PYnS6/0CZvwYO04Q7MFMf+psrosW3OO", + "yUfH1l4S/7A4/gMbHWZr6+snC4KTHtQjdsA9KFc0oR8Nq9ONUKloZFKOcLF2708LptofslOihH4HQ2pb", + "khhEA9AutzLBo61hPhjsRhkF9LddAoMDhtf8OIUej87emTRUknKx6A6Z/gc0/PbwzHh3J9haE7yB2sLJ", + "6GTr9ZoA53NYpn/jCEEzwZXoBcEN/+YSvDnGSOMZkg1HlGerVCWe/elDWK0E982u8DjtCgDyVMxmowD2", + "cmhcYRvCnCd5qv9h/jhZh2umcDR7D69+NdKuGc7abtwEH8WhtHOKialt+SBOD7NgjzVmVS+cmwIIMZVo", + "wOAtcKj+jNT95c33/jp+he5Ou6KubuxXc7bu++azY3AIG/56PJZjbijNzUTx1danK0ybrU8/Jjy6lBaK", + "xTcbar0N8NX1jyUetnURgpgAmaHIQhgZoCwiu0NWM0AaxB+JMFJEpJThZAvmbBoBZG9nxcJzTiFBO4I8", + "lZ6kMWAmJQDfDfB3ejZgqHINeB5daStr+e/4zkjF0ZhEPCUO7XwzpLr9DVP1ExdV6PKvhS++9dYfIAEx", + "BXv7GrT25h4/C739FF9DqHScW4eyG9HGS17+aExBXQR7M+zsDuSw00XDzk467OgdOMJgQsUK7aOUslwR", + "2UfHxr4FKbhPBkiSiLNYOtB1Z8HbHcimhFxDlg3ZnU/gu/sUeyxVwVK+sZ2E2IN+D+nvIWkHbfgHzp7J", + "uAuHLkY8V8bcb8+VfSsmCswjm/fuq/XOyDfdvg0n/5s9vhUeBbus2aW39YazZ7mckWaT2ytTyChXYwDz", + "dsVF5Qz9nY9lFzFyZazhQqr+Et/TX5+ZDu6j0IDu6iZFBuzcv1UYaFFhoFyrMFijCbDUV7KjDoPYSK4z", + "LhSgONpce0NDoEkAcgSPcIJeH50MWaRZkYEWFCTlwJ0sHrq5hQ//do5eHL3pomModIl+zsebffSaJQtX", + "btz4aIbMSGKGeUWYobGhWhKHrmczdqCeuwwW1x08UOVoczICnhW3Vy5IvNuZERyDRPJH5xU3nQVQh9+8", + "0gcIgH/Nl8W2d1YKH503RIlF73CiiFhu9tTmSbECM8Ne0g6CzgpuBvhSdygd8lrZp5ENDDTG7k4ngJTx", + "6VvRh7svkHo/XjITJ2LK7Y1zQBplkGSA48XjimWSM1QwxxAL9K/romxCU5aw5WUrFQzosiny+ysyua/k", + "XRVs+X/X0wUzfbSOpqyyT5qIi3Iraz29Ljl4ZuCQraMqwhmOqFp0EU4Se0fZm6CISOkV4u9YEHwZ8yvW", + "H7I3RaEXm9CLjs7edZ2jFsVUXpoWrC+2j17PiZD5uBgcgoNmvMaw5iQeMsVRhJMoT7S4QSYTEkEuLtRv", + "kQ2+3GIonTs8O2UnwWIzXlR7/uhq3IVpAnavJIs6xW2Zrd4SJEowTZvBx62gBgGHEGow1o1yhiibJDak", + "KhJcSmSb6pGETuk4sQFCso/ezgiSOCVDliWYMSJQLk1UvB56LxNEytwkeOsGAKTXUFQXlcCCmeDKhiYk", + "nAtpogk0hb8/RVKRbAWZvTEtn8Kc70i2NY3bnh7ISF0bQ7MpxL6C9IYYSjELrukoT1wA472GopsBPbSU", + "+FgO/ltBp1Mi9KnAhsmacDxzrN1ymkNfyVhurHd5XrzVrt5l0aqXlehl7K0EhhuVWNtx52ZRf4HOL2kj", + "dqB9dLMs4l/0Ry37rmarhgdhH33mLEOlO/8dq2See0mCbQ1YJYU/NnOSN/LKUa0k2q6H1WqdWXuXma6t", + "8bMeDDbrMaNl4Ur6bJPC+/URwuB+UR7uu8ja46atCtpVRTdtSPlfj6b/VVDg3cDoPzDKyS1g9L+qvHvA", + "OX84/JPgQX2oPPqK79kV2/3TI+HfVfq8gcMHOLam9HnD9Wzw6kpF6b19p52aZFv8M0nwNt7xBvK7W/Zv", + "Wn8LlcFbrHUuaE3wJM3UwgW0WV9lGXQm6UfSb3AEF3Grd+cKvkVI55cjD0enjQGdf87a+A8SM2pLB1KJ", + "To4DRecfGcagf+YqF8uWvnV6WEQzOifNRvfqCbZLlAnSy3gGzpXYLJhdD3eXKSz604/INm8xV+2/oPYk", + "QPWTGMVUkEglC1MHVHME08d3EgmuNQF4zsWiOUrEHJGfBE8P7WzW3If2TFljWBlnmC56MVa4N3fcZoUJ", + "7TOiO108pWZ4iDL08ke0Qa6VMBUu0ERrPohOiiUl1xEhsQSa3PQHvD1osGzSj2Q0HbcZ5YpaJa9tLRgU", + "5VLx1O39yTHagNpnU8L0XmhRfwKSbCb4nMYkroyxM+eJWdXthgW9qd1VCxVF4TqnXJjBPYgM0+ZCmn6k", + "WZUtFCExY8owDG5tVZDqmTJJ/Lo/TJkLwLF75Ebx7Qqzmt+GU3Y0JUIdTruIinMD8bz57Zp7zNecnwzl", + "7rTKbefCc1Ybr9vlR7VMW7qLwg9F7tz9mq3ffz0pPVQ+ymweazqfFwppk9n86yLBwf3dD/dtLn//iFNA", + "XxKnfHumcmhAtxgimFcQ0x2TOUl4lkI9dHi30+3kIukcdGZKZQdbWxD7PeNSHew9f7rb+fTh0/8fAAD/", + "//uqpNOh8AEA", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/openapi.yaml b/openapi.yaml index fa0e1381f..c33c796d8 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -1757,8 +1757,12 @@ components: example: "L40S-1Q" mdev_uuid: type: string - description: mdev device UUID + description: mdev device UUID (mdev hosts only) example: "aa618089-8b16-4d01-a136-25a0f3c73123" + device_path: + type: string + description: sysfs path of the assigned vGPU device + example: "/sys/bus/pci/devices/0000:82:00.4" GPUProfile: type: object From 4fe3ad926e51be92698b9ba4a7bbccd111ccfab0 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Tue, 11 Aug 2026 17:53:21 +0000 Subject: [PATCH 32/76] Surface retained rollback assignments from start as vgpu_cleanup_pending When a later start step failed and rollback could not destroy the freshly created vGPU, cleanupStartVGPU retained the assignment on disk but startInstance returned the original failure untyped, so the API reported a generic error instead of vgpu_cleanup_pending with the retained-assignment guidance. Mirror create's named-return wrap: cleanupStartVGPU reports retention state and start wraps the returned error in VGPUCleanupPendingError. --- lib/instances/start.go | 12 +++++++++++- lib/instances/vgpu.go | 10 +++++++++- lib/instances/vgpu_test.go | 4 ++++ 3 files changed, 24 insertions(+), 2 deletions(-) diff --git a/lib/instances/start.go b/lib/instances/start.go index b44a68d50..1045829a1 100644 --- a/lib/instances/start.go +++ b/lib/instances/start.go @@ -116,6 +116,16 @@ func (m *manager) startInstance( } // Setup cleanup stack for automatic rollback on errors + // Registered before cu.Clean so it runs after cleanup and can report a + // vGPU assignment that rollback failed to destroy, matching create's + // vgpu_cleanup_pending contract. + vgpuRetained := false + vgpuRetentionPersisted := false + defer func() { + if retErr != nil && vgpuRetained { + retErr = &VGPUCleanupPendingError{InstanceID: id, Retained: vgpuRetentionPersisted, Err: retErr} + } + }() cu := cleanup.Make(func() {}) defer cu.Clean() @@ -189,7 +199,7 @@ func (m *manager) startInstance( log.InfoContext(ctx, "created vGPU", "instance_id", id, "profile", stored.GPUProfile, "uuid", device.MdevUUID) // Add vGPU cleanup to stack cu.Add(func() { - m.cleanupStartVGPU(ctx, id, device, assignedAt, rollbackMeta) + vgpuRetained, vgpuRetentionPersisted = m.cleanupStartVGPU(ctx, id, device, assignedAt, rollbackMeta) }) // Checked after the cleanup handler is registered so rejection // releases the device through the normal rollback. diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index 8c6351742..1e3dcc356 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -105,7 +105,12 @@ func clearStoredVGPUDevice(stored *StoredMetadata) { // start. The snapshot is also a shallow copy (Phases shares its map), so it // must be persisted before any Phases.Record on the live struct. Violating // either invariant requires switching to targeted field restores. -func (m *manager) cleanupStartVGPU(ctx context.Context, instanceID string, device *devices.VGPUDevice, assignedAt time.Time, rollbackMeta metadata) { +// +// It reports whether the assignment was retained after a failed destroy and +// whether that retention record was persisted, so start can surface the +// pending cleanup as a typed error like create does. +func (m *manager) cleanupStartVGPU(ctx context.Context, instanceID string, device *devices.VGPUDevice, assignedAt time.Time, rollbackMeta metadata) (retained, persisted bool) { + logger.FromContext(ctx).DebugContext(ctx, "destroying vGPU on cleanup", "instance_id", instanceID, "uuid", device.MdevUUID) assignment := devices.VGPUAssignment{ Framework: device.Framework, DevicePath: device.SysfsPath, @@ -117,6 +122,7 @@ func (m *manager) cleanupStartVGPU(ctx context.Context, instanceID string, devic if releaseErr != nil { logger.FromContext(ctx).WarnContext(ctx, "failed to destroy vGPU on cleanup", "instance_id", instanceID, "error", releaseErr) setStoredVGPUDevice(&cleanupMeta.StoredMetadata, device, assignedAt) + retained = true } if err := m.saveMetadata(&cleanupMeta); err != nil { message := "failed to save metadata after vGPU cleanup" @@ -124,7 +130,9 @@ func (m *manager) cleanupStartVGPU(ctx context.Context, instanceID string, devic message = "failed to retain vGPU assignment metadata after cleanup failure" } logger.FromContext(ctx).ErrorContext(ctx, message, "instance_id", instanceID, "error", err) + return retained, false } + return retained, retained } func (m *manager) releaseStoredVGPU(ctx context.Context, stored *StoredMetadata) error { diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index d466f1883..7668ff90c 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -389,6 +389,10 @@ func TestStartRollbackRetainsVGPUAssignmentAfterFailedDestroy(t *testing.T) { t.Setenv("TMPDIR", filepath.Join(t.TempDir(), "missing")) _, err = m.startInstance(context.Background(), id, StartInstanceRequest{Entrypoint: []string{"new-entrypoint"}}) require.Error(t, err) + var pending *VGPUCleanupPendingError + require.ErrorAs(t, err, &pending, "a retained rollback assignment must surface as vgpu_cleanup_pending") + assert.Equal(t, id, pending.InstanceID) + assert.True(t, pending.Retained) stored, err := m.loadMetadata(id) require.NoError(t, err) From 01f754b9292f51306620c41eea3b8c404cfa3cfc Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Tue, 11 Aug 2026 18:07:37 +0000 Subject: [PATCH 33/76] Report retention as persisted when the mid-start save survives When start rollback fails to destroy a vGPU and the cleanup metadata save also fails, the assignment may still be on disk from the mid-start save. Reporting Retained: false then misdirects callers to wait for startup reconcile when delete or a retried start can already release it. Check whether the surviving record still points at the device, matching create's retention-survives check. --- lib/instances/vgpu.go | 12 +++++++++++- lib/instances/vgpu_test.go | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index 1e3dcc356..330e2d820 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -130,7 +130,17 @@ func (m *manager) cleanupStartVGPU(ctx context.Context, instanceID string, devic message = "failed to retain vGPU assignment metadata after cleanup failure" } logger.FromContext(ctx).ErrorContext(ctx, message, "instance_id", instanceID, "error", err) - return retained, false + if !retained { + return false, false + } + // The mid-start save may already have persisted this assignment, in + // which case the on-disk record still points at the device and + // delete or a retried start can release it (matching create's + // retention-survives check). + if meta, loadErr := m.loadMetadata(instanceID); loadErr == nil && storedVGPUDevicePath(&meta.StoredMetadata) == device.SysfsPath { + return true, true + } + return true, false } return retained, retained } diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index 7668ff90c..b34fe242a 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -405,6 +405,38 @@ func TestStartRollbackRetainsVGPUAssignmentAfterFailedDestroy(t *testing.T) { assert.Empty(t, stored.Entrypoint) } +func TestCleanupStartVGPUReportsRetainedWhenMidStartSaveSurvives(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("root bypasses directory permissions") + } + + m, id := newStartRollbackVGPUManager(t, func(context.Context, devices.VGPUAssignment) error { + return errors.New("destroy failed") + }) + device := devices.VGPUDevice{ + Framework: devices.VGPUFrameworkVendorVFIO, + SysfsPath: "/sys/bus/pci/devices/0000:82:00.4", + } + assignedAt := time.Now().UTC() + + // The mid-start save already persisted the assignment. + meta, err := m.loadMetadata(id) + require.NoError(t, err) + rollbackMeta := *meta + setStoredVGPUDevice(&meta.StoredMetadata, &device, assignedAt) + require.NoError(t, m.saveMetadata(meta)) + + // The cleanup save fails, but the surviving on-disk record still points + // at the device, so retention must be reported as persisted. + instanceDir := filepath.Dir(m.paths.InstanceMetadata(id)) + require.NoError(t, os.Chmod(instanceDir, 0o555)) + t.Cleanup(func() { _ = os.Chmod(instanceDir, 0o755) }) + + retained, persisted := m.cleanupStartVGPU(context.Background(), id, &device, assignedAt, rollbackMeta) + assert.True(t, retained) + assert.True(t, persisted, "a surviving mid-start save keeps the assignment recoverable via delete") +} + func TestCleanupStartVGPURestoresMetadataAfterBootFailure(t *testing.T) { m := &manager{ paths: paths.New(t.TempDir()), From 18388464f864ddd0afe5daaf57f0530ebb949b6d Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Tue, 11 Aug 2026 19:07:40 +0000 Subject: [PATCH 34/76] Leave vGPU hypervisor selection to callers Drop the vendor-VFIO-on-Cloud-Hypervisor rejection from create and start, restoring the phase-0 decision that hypervisor selection is caller policy: production callers pin vGPU instances to QEMU, and the Cloud Hypervisor limitation stays documented in lib/devices/GPU.md. --- lib/instances/create.go | 6 ------ lib/instances/start.go | 6 ------ lib/instances/vgpu.go | 13 ------------- lib/instances/vgpu_test.go | 29 ----------------------------- 4 files changed, 54 deletions(-) diff --git a/lib/instances/create.go b/lib/instances/create.go index 23fd534b1..a6cce101a 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -357,12 +357,6 @@ func (m *manager) createInstance( } } }) - // Checked after the cleanup handler is registered so rejection - // releases the device through the normal rollback. - if err := validateVGPUHypervisorCompat(gpuDevice.Framework, hvType); err != nil { - log.ErrorContext(ctx, "unsupported vGPU hypervisor combination", "instance_id", id, "framework", gpuDevice.Framework, "hypervisor", hvType) - return nil, err - } } if len(req.Devices) > 0 && m.deviceManager != nil { diff --git a/lib/instances/start.go b/lib/instances/start.go index 1045829a1..3e164b398 100644 --- a/lib/instances/start.go +++ b/lib/instances/start.go @@ -201,12 +201,6 @@ func (m *manager) startInstance( cu.Add(func() { vgpuRetained, vgpuRetentionPersisted = m.cleanupStartVGPU(ctx, id, device, assignedAt, rollbackMeta) }) - // Checked after the cleanup handler is registered so rejection - // releases the device through the normal rollback. - if err := validateVGPUHypervisorCompat(device.Framework, stored.HypervisorType); err != nil { - log.ErrorContext(ctx, "unsupported vGPU hypervisor combination", "instance_id", id, "framework", device.Framework, "hypervisor", stored.HypervisorType) - return nil, err - } if err := m.saveMetadata(meta); err != nil { log.ErrorContext(ctx, "failed to save metadata after vGPU creation", "instance_id", id, "error", err) return nil, fmt.Errorf("save metadata after vGPU creation: %w", err) diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index 330e2d820..88ae217da 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -8,7 +8,6 @@ import ( "time" "github.com/kernel/hypeman/lib/devices" - "github.com/kernel/hypeman/lib/hypervisor" "github.com/kernel/hypeman/lib/logger" ) @@ -64,18 +63,6 @@ func retainedVGPUFromCreateError(instanceID string, assignedAt time.Time, err er } } -// validateVGPUHypervisorCompat rejects the one proven-broken combination: -// vendor VFIO vGPUs boot but are non-functional on Cloud Hypervisor (upstream -// cloud-hypervisor#7572), and the wedged VM then blocks the VF release until -// startup reconcile. Hypervisor selection otherwise remains caller policy; -// mdev on Cloud Hypervisor keeps working. See lib/devices/GPU.md. -func validateVGPUHypervisorCompat(framework devices.VGPUFramework, hvType hypervisor.Type) error { - if framework == devices.VGPUFrameworkVendorVFIO && hvType == hypervisor.TypeCloudHypervisor { - return fmt.Errorf("%w: vendor VFIO vGPUs are not functional on cloud-hypervisor, use qemu", ErrInvalidRequest) - } - return nil -} - func (m *manager) destroyVGPUAssignment(ctx context.Context, assignment devices.VGPUAssignment) error { destroy := m.destroyVGPU if destroy == nil { diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index b34fe242a..1bdb0fb97 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -319,35 +319,6 @@ func TestStartDoesNotRestrictVGPUHypervisor(t *testing.T) { assert.ErrorIs(t, err, cause) } -func TestValidateVGPUHypervisorCompat(t *testing.T) { - t.Parallel() - - err := validateVGPUHypervisorCompat(devices.VGPUFrameworkVendorVFIO, hypervisor.TypeCloudHypervisor) - require.ErrorIs(t, err, ErrInvalidRequest) - assert.NoError(t, validateVGPUHypervisorCompat(devices.VGPUFrameworkVendorVFIO, hypervisor.TypeQEMU)) - assert.NoError(t, validateVGPUHypervisorCompat(devices.VGPUFrameworkMdev, hypervisor.TypeCloudHypervisor)) -} - -func TestStartRejectsVendorVFIOOnCloudHypervisor(t *testing.T) { - var destroyed []devices.VGPUAssignment - m, id := newStartRollbackVGPUManager(t, func(_ context.Context, assignment devices.VGPUAssignment) error { - destroyed = append(destroyed, assignment) - return nil - }) - meta, err := m.loadMetadata(id) - require.NoError(t, err) - meta.HypervisorType = hypervisor.TypeCloudHypervisor - require.NoError(t, m.saveMetadata(meta)) - - _, err = m.startInstance(context.Background(), id, StartInstanceRequest{}) - require.ErrorIs(t, err, ErrInvalidRequest) - - require.Len(t, destroyed, 1, "the rejected vGPU must be released by rollback") - stored, err := m.loadMetadata(id) - require.NoError(t, err) - assert.Empty(t, stored.GPUDevicePath, "no assignment may be persisted for a rejected combination") -} - func TestStartRollbackClearsVGPUAssignmentAfterSuccessfulDestroy(t *testing.T) { var destroyed []devices.VGPUAssignment m, id := newStartRollbackVGPUManager(t, func(_ context.Context, assignment devices.VGPUAssignment) error { From b79b13e26e55aabef14199915aa5402c0d80bcb6 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Tue, 11 Aug 2026 19:21:39 +0000 Subject: [PATCH 35/76] Carry identity fields into the create-pending retention stub retainedVGPUFromCreateError built a GPU-fields-only stub, so the retained record from a failed device-layer cleanup listed nameless and, with GPUProfile empty, the API hid its gpu block including device_path. The caller now supplies the identity fields and the stub picks up the pending device's profile. --- lib/instances/create.go | 16 +++++++++++++++- lib/instances/vgpu.go | 19 +++++++++++-------- lib/instances/vgpu_test.go | 6 ++++-- 3 files changed, 30 insertions(+), 11 deletions(-) diff --git a/lib/instances/create.go b/lib/instances/create.go index a6cce101a..07abfd4b6 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -312,7 +312,21 @@ func (m *manager) createInstance( log.InfoContext(ctx, "creating vGPU", "instance_id", id, "profile", req.GPU.Profile) gpuDevice, err = m.createVGPUDevice(ctx, req.GPU.Profile, id) if err != nil { - retainedVGPU = retainedVGPUFromCreateError(id, m.nowUTC(), err) + stub := StoredMetadata{ + Id: id, + Name: req.Name, + Image: req.Image, + ResolvedImage: resolvedImageRef, + Platform: imageInfo.Platform, + CreatedAt: time.Now(), + HypervisorType: hvType, + HypervisorVersion: hvVersion, + DataDir: m.paths.InstanceDir(id), + } + if starterErr == nil { + stub.SocketPath = m.paths.InstanceSocket(id, starter.SocketName()) + } + retainedVGPU = retainedVGPUFromCreateError(stub, m.nowUTC(), err) log.ErrorContext(ctx, "failed to create vGPU", "profile", req.GPU.Profile, "error", err) return nil, wrapCreateVGPUErr(req.GPU.Profile, err) } diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index 88ae217da..4caa365bb 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -49,18 +49,21 @@ func vgpuDevicePendingCleanup(err error) (*devices.VGPUDevice, bool) { return &pending.Device, true } -func retainedVGPUFromCreateError(instanceID string, assignedAt time.Time, err error) *StoredMetadata { +// retainedVGPUFromCreateError fills stub with the pending device's assignment +// fields when err carries a failed device-layer cleanup. The caller provides +// identity fields on stub so the retained record lists as a recognizable, +// deletable instance. +func retainedVGPUFromCreateError(stub StoredMetadata, assignedAt time.Time, err error) *StoredMetadata { device, ok := vgpuDevicePendingCleanup(err) if !ok { return nil } - return &StoredMetadata{ - Id: instanceID, - GPUFramework: device.Framework, - GPUDevicePath: device.SysfsPath, - GPUMdevUUID: device.MdevUUID, - GPUAssignedAt: &assignedAt, - } + stub.GPUProfile = device.ProfileName + stub.GPUFramework = device.Framework + stub.GPUDevicePath = device.SysfsPath + stub.GPUMdevUUID = device.MdevUUID + stub.GPUAssignedAt = &assignedAt + return &stub } func (m *manager) destroyVGPUAssignment(ctx context.Context, assignment devices.VGPUAssignment) error { diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index 1bdb0fb97..e5694ef3f 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -143,9 +143,11 @@ func TestVGPUDevicePendingCleanup(t *testing.T) { assert.Equal(t, device, *actual) assignedAt := time.Now().UTC() - retained := retainedVGPUFromCreateError("inst-1", assignedAt, wrapped) + retained := retainedVGPUFromCreateError(StoredMetadata{Id: "inst-1", Name: "named", Image: "img"}, assignedAt, wrapped) require.NotNil(t, retained) assert.Equal(t, "inst-1", retained.Id) + assert.Equal(t, "named", retained.Name, "identity fields must survive into the retention stub") + assert.Equal(t, "img", retained.Image) assert.Equal(t, device.Framework, retained.GPUFramework) assert.Equal(t, device.SysfsPath, retained.GPUDevicePath) assert.Equal(t, assignedAt, *retained.GPUAssignedAt) @@ -153,7 +155,7 @@ func TestVGPUDevicePendingCleanup(t *testing.T) { actual, ok = vgpuDevicePendingCleanup(cause) assert.False(t, ok) assert.Nil(t, actual) - assert.Nil(t, retainedVGPUFromCreateError("inst-1", assignedAt, cause)) + assert.Nil(t, retainedVGPUFromCreateError(StoredMetadata{Id: "inst-1"}, assignedAt, cause)) } type startRetentionNetworkManager struct { From 84f821a671de69d2d2f1083683469617678cae3c Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Tue, 11 Aug 2026 19:39:32 +0000 Subject: [PATCH 36/76] Grace recent dead-PID claims in the release scan like reconcile does Startup reconcile protects an assignment whose PID is absent or stale for a bounded grace window, but the release-side claim scan treated a dead PID as unclaimed immediately. Align the two guards: a recent assignment whose recorded hypervisor is not running fails the scan closed so the requester retains and retries, and past the grace window the dead claim no longer blocks the release. --- lib/instances/vgpu.go | 8 +++++++- lib/instances/vgpu_test.go | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index 4caa365bb..2f9cfa7d1 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -110,7 +110,7 @@ func (m *manager) cleanupStartVGPU(ctx context.Context, instanceID string, devic cleanupMeta := rollbackMeta releaseErr := m.destroyVGPUAssignment(ctx, assignment) if releaseErr != nil { - logger.FromContext(ctx).WarnContext(ctx, "failed to destroy vGPU on cleanup", "instance_id", instanceID, "error", releaseErr) + logger.FromContext(ctx).WarnContext(ctx, "failed to destroy vGPU on cleanup", "instance_id", instanceID, "uuid", device.MdevUUID, "error", releaseErr) setStoredVGPUDevice(&cleanupMeta.StoredMetadata, device, assignedAt) retained = true } @@ -208,6 +208,12 @@ func (m *manager) vgpuAssignmentClaimedByLiveInstance(ctx context.Context, exclu if pid > 0 { return true, nil } + // A dead PID with a recent assignment gets the same bounded grace as + // startup reconcile protection, so the two guards agree in the + // fail-closed direction while a mid-boot claimant hydrates. + if stored.GPUAssignedAt != nil && time.Since(*stored.GPUAssignedAt) < VGPUAssignmentStartupGracePeriod { + return false, fmt.Errorf("cannot confirm liveness of recent vGPU claimant %s on %s: recorded hypervisor is not running", id, devicePath) + } } return false, nil } diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index e5694ef3f..d5e988a43 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -528,6 +528,38 @@ func TestVGPUAssignmentClaimedByLiveInstanceIgnoresStaleNilPIDClaim(t *testing.T assert.False(t, claimed) } +func TestVGPUAssignmentClaimedByLiveInstanceGracesRecentDeadPIDClaim(t *testing.T) { + m := &manager{paths: paths.New(t.TempDir())} + claimantID := "claimant-dead-pid" + require.NoError(t, m.ensureDirectories(claimantID)) + deadPID := 1<<22 - 1 + require.False(t, ProcessExists(deadPID)) + assignedAt := time.Now().UTC() + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: claimantID, + HypervisorPID: &deadPID, + GPUFramework: devices.VGPUFrameworkVendorVFIO, + GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", + GPUAssignedAt: &assignedAt, + }})) + + // Same bounded grace as startup reconcile: a recent claim whose PID is + // dead fails closed instead of being treated as unclaimed. + _, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "requester", "/sys/bus/pci/devices/0000:82:00.4") + require.Error(t, err) + + // Past the grace period the dead claim no longer blocks the release. + stale := assignedAt.Add(-2 * VGPUAssignmentStartupGracePeriod) + meta, err := m.loadMetadata(claimantID) + require.NoError(t, err) + meta.GPUAssignedAt = &stale + require.NoError(t, m.saveMetadata(meta)) + + claimed, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "requester", "/sys/bus/pci/devices/0000:82:00.4") + require.NoError(t, err) + assert.False(t, claimed) +} + func TestVGPUAssignmentClaimedByLiveInstanceIgnoresDeadClaim(t *testing.T) { t.Parallel() From af8d527a6399512681d68638d7136697220d6e0d Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Thu, 13 Aug 2026 18:57:32 +0000 Subject: [PATCH 37/76] Reject start on vGPU retention records A failed create whose vGPU release also failed persists a delete-only retention stub with no boot configuration. The stub derives as Stopped, so start would release the retained VF and then try to boot the incomplete record. Mark the stub with GPURetainedForCleanup and reject start with invalid_state guidance pointing at delete, which retries the release. --- lib/instances/create.go | 31 ++++++++++++++-------------- lib/instances/lifecycle_noop_test.go | 20 ++++++++++++++++++ lib/instances/start.go | 7 +++++++ lib/instances/types.go | 4 ++++ lib/instances/vgpu_test.go | 2 ++ 5 files changed, 49 insertions(+), 15 deletions(-) diff --git a/lib/instances/create.go b/lib/instances/create.go index 07abfd4b6..e7ab3b6ad 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -687,21 +687,22 @@ func (m *manager) cleanupFailedCreate(ctx context.Context, id string, retainedVG // deletable record rather than a nameless phantom, but drop resource // claims (network, volumes, devices) that rollback already released. retained := StoredMetadata{ - Id: id, - Name: retainedVGPU.Name, - Image: retainedVGPU.Image, - ResolvedImage: retainedVGPU.ResolvedImage, - Platform: retainedVGPU.Platform, - CreatedAt: retainedVGPU.CreatedAt, - HypervisorType: retainedVGPU.HypervisorType, - HypervisorVersion: retainedVGPU.HypervisorVersion, - SocketPath: retainedVGPU.SocketPath, - DataDir: retainedVGPU.DataDir, - GPUProfile: retainedVGPU.GPUProfile, - GPUFramework: retainedVGPU.GPUFramework, - GPUDevicePath: retainedVGPU.GPUDevicePath, - GPUMdevUUID: retainedVGPU.GPUMdevUUID, - GPUAssignedAt: retainedVGPU.GPUAssignedAt, + Id: id, + Name: retainedVGPU.Name, + Image: retainedVGPU.Image, + ResolvedImage: retainedVGPU.ResolvedImage, + Platform: retainedVGPU.Platform, + CreatedAt: retainedVGPU.CreatedAt, + HypervisorType: retainedVGPU.HypervisorType, + HypervisorVersion: retainedVGPU.HypervisorVersion, + SocketPath: retainedVGPU.SocketPath, + DataDir: retainedVGPU.DataDir, + GPUProfile: retainedVGPU.GPUProfile, + GPUFramework: retainedVGPU.GPUFramework, + GPUDevicePath: retainedVGPU.GPUDevicePath, + GPUMdevUUID: retainedVGPU.GPUMdevUUID, + GPUAssignedAt: retainedVGPU.GPUAssignedAt, + GPURetainedForCleanup: true, } if err := m.saveMetadata(&metadata{StoredMetadata: retained}); err != nil { log.ErrorContext(ctx, "failed to retain vGPU assignment metadata after cleanup failure", "instance_id", id, "error", err) diff --git a/lib/instances/lifecycle_noop_test.go b/lib/instances/lifecycle_noop_test.go index c74d352fd..e440e7e4a 100644 --- a/lib/instances/lifecycle_noop_test.go +++ b/lib/instances/lifecycle_noop_test.go @@ -324,6 +324,26 @@ func TestStartPersistsStaleVGPUReleaseImmediately(t *testing.T) { assert.Equal(t, "NVIDIA L40S-2Q", stored.GPUProfile, "profile is kept for the next start") } +func TestStartRejectsVGPURetentionRecord(t *testing.T) { + m, id := newLifecycleNoopManagerWithInstance(t, StateStopped, time.Now().UTC()) + meta, err := m.loadMetadata(id) + require.NoError(t, err) + meta.GPUProfile = "NVIDIA L40S-2Q" + meta.GPUFramework = devices.VGPUFrameworkNone + meta.GPUDevicePath = "/sys/bus/pci/devices/0000:82:00.4" + meta.GPURetainedForCleanup = true + require.NoError(t, m.saveMetadata(meta)) + + _, err = m.StartInstance(context.Background(), id, StartInstanceRequest{}) + require.ErrorIs(t, err, ErrInvalidState) + require.ErrorContains(t, err, "delete it to release the assignment") + + // The retained assignment must survive the rejected start for delete. + stored, err := m.loadMetadata(id) + require.NoError(t, err) + assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", stored.GPUDevicePath) +} + func TestStopStoppedInstanceReleasesRetainedVGPU(t *testing.T) { m, id := newLifecycleNoopManagerWithInstance(t, StateStopped, time.Now().UTC()) meta, err := m.loadMetadata(id) diff --git a/lib/instances/start.go b/lib/instances/start.go index 3e164b398..8914f2ae1 100644 --- a/lib/instances/start.go +++ b/lib/instances/start.go @@ -47,6 +47,13 @@ func (m *manager) startInstance( log.ErrorContext(ctx, "invalid state for start", "instance_id", id, "state", inst.State) return nil, fmt.Errorf("%w: cannot start from state %s, must be Stopped", ErrInvalidState, inst.State) } + if stored.GPURetainedForCleanup { + // A delete-only retention stub from a failed create: it carries no + // boot configuration, so starting it would release the retained VF + // and then boot an incomplete record. Delete retries the release. + log.ErrorContext(ctx, "refusing to start vGPU retention record", "instance_id", id) + return nil, fmt.Errorf("%w: instance retains a vGPU assignment from a failed create and has no boot configuration; delete it to release the assignment", ErrInvalidState) + } // Release any assignment retained by an earlier failed release and // persist the cleared fields immediately, so a failure later in start diff --git a/lib/instances/types.go b/lib/instances/types.go index a264498fd..ab41178cd 100644 --- a/lib/instances/types.go +++ b/lib/instances/types.go @@ -156,6 +156,10 @@ type StoredMetadata struct { GPUDevicePath string GPUMdevUUID string // populated for mdev-backed vGPUs GPUAssignedAt *time.Time // set before hypervisor startup to bound crash recovery protection + // GPURetainedForCleanup marks a delete-only retention stub written when a + // failed create could not release its vGPU: the record has no boot + // configuration, so only delete (which retries the release) may act on it. + GPURetainedForCleanup bool // Command overrides (like docker run ) Entrypoint []string // Override image entrypoint (nil = use image default) diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index d5e988a43..a6fe4e49a 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -57,6 +57,8 @@ func TestCleanupFailedCreateRetainsVGPUAssignment(t *testing.T) { assert.False(t, retained.NetworkEnabled) assert.Empty(t, retained.IP) assert.Empty(t, retained.Volumes) + // The stub has no boot configuration, so it is marked delete-only. + assert.True(t, retained.GPURetainedForCleanup) } func TestCleanupFailedCreateDeletesDataWithoutRetainedVGPU(t *testing.T) { From ca6f98d2aadf6915053681ceb55e6369a5ad7c7f Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Thu, 13 Aug 2026 19:05:38 +0000 Subject: [PATCH 38/76] Make vGPU retention records fully delete-only A fork or snapshot of a failed-create retention stub could never boot: the stub has no boot configuration, and clearing the delete-only marker on the child would only produce a startable-but-broken record that recreates a vGPU from GPUProfile with incomplete metadata. Reject fork and snapshot of retention stubs with the same invalid_state guidance as start, so delete (which retries the release) is the only action on them. --- lib/instances/fork.go | 6 ++++++ lib/instances/fork_test.go | 25 +++++++++++++++++++++++++ lib/instances/snapshot.go | 7 +++++++ lib/instances/snapshot_test.go | 25 +++++++++++++++++++++++++ 4 files changed, 63 insertions(+) diff --git a/lib/instances/fork.go b/lib/instances/fork.go index e0c778860..266299e96 100644 --- a/lib/instances/fork.go +++ b/lib/instances/fork.go @@ -219,6 +219,12 @@ func (m *manager) forkInstanceFromStoppedOrStandby(ctx context.Context, id strin default: return nil, false, fmt.Errorf("%w: cannot fork from state %s (must be Stopped or Standby)", ErrInvalidState, source.State) } + if stored.GPURetainedForCleanup { + // A delete-only retention stub from a failed create has no boot + // configuration, so a fork of it could never boot. Delete the stub to + // release its retained vGPU assignment. + return nil, false, fmt.Errorf("%w: instance retains a vGPU assignment from a failed create and has no boot configuration; delete it to release the assignment", ErrInvalidState) + } if !supportValidated { if err := m.validateForkSupport(ctx, stored.HypervisorType); err != nil { diff --git a/lib/instances/fork_test.go b/lib/instances/fork_test.go index 26bc6cfcb..ef802d1ad 100644 --- a/lib/instances/fork_test.go +++ b/lib/instances/fork_test.go @@ -63,6 +63,31 @@ func TestForkInstanceClearsVGPUAssignment(t *testing.T) { assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", source.GPUDevicePath) } +func TestForkInstanceRejectsVGPURetentionRecord(t *testing.T) { + manager, _ := setupTestManager(t) + ctx := context.Background() + hvType := hypervisor.Type("fork-vgpu-retention-test") + hypervisor.RegisterCapabilities(hvType, hypervisor.Capabilities{SupportsConcurrentForkPrepare: true}) + manager.vmStarters[hvType] = concurrentForkPrepareTestStarter{} + + sourceID := "fork-vgpu-retention-source" + createStoppedSnapshotSourceFixture(t, manager, sourceID, sourceID, hvType) + + meta, err := manager.loadMetadata(sourceID) + require.NoError(t, err) + meta.GPUProfile = "NVIDIA L40S-2Q" + meta.GPUFramework = devices.VGPUFramework("future-framework") + meta.GPUDevicePath = "/sys/bus/pci/devices/0000:82:00.4" + meta.GPURetainedForCleanup = true + require.NoError(t, manager.saveMetadata(meta)) + + // The delete-only retention stub has no boot configuration, so a fork of + // it could never boot; only delete may act on it. + _, err = manager.ForkInstance(ctx, sourceID, ForkInstanceRequest{Name: "fork-vgpu-retention-copy"}) + require.ErrorIs(t, err, ErrInvalidState) + require.ErrorContains(t, err, "delete it to release the assignment") +} + func TestForkInstance_VZStoppedSourceSupported(t *testing.T) { t.Parallel() manager, _ := setupTestManager(t) diff --git a/lib/instances/snapshot.go b/lib/instances/snapshot.go index 48c51328a..297c7ef03 100644 --- a/lib/instances/snapshot.go +++ b/lib/instances/snapshot.go @@ -66,6 +66,13 @@ func (m *manager) createSnapshot(ctx context.Context, id string, req CreateSnaps inst := m.toInstance(ctx, meta) stored := &meta.StoredMetadata + if stored.GPURetainedForCleanup { + // A delete-only retention stub from a failed create has no boot + // configuration, so a snapshot of it could never be restored or + // forked into a bootable instance. Delete the stub to release its + // retained vGPU assignment. + return nil, fmt.Errorf("%w: instance retains a vGPU assignment from a failed create and has no boot configuration; delete it to release the assignment", ErrInvalidState) + } if err := validateForkVolumeSafety(stored.Volumes); err != nil { return nil, fmt.Errorf("%w: snapshot requires readonly volume attachments: %v", ErrNotSupported, err) } diff --git a/lib/instances/snapshot_test.go b/lib/instances/snapshot_test.go index a92763b28..f8c022fe9 100644 --- a/lib/instances/snapshot_test.go +++ b/lib/instances/snapshot_test.go @@ -52,6 +52,31 @@ func TestForkSnapshotClearsVGPUAssignment(t *testing.T) { assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", source.GPUDevicePath) } +func TestCreateSnapshotRejectsVGPURetentionRecord(t *testing.T) { + mgr, _ := setupTestManager(t) + ctx := context.Background() + + sourceID := "snapshot-vgpu-retention" + createStoppedSnapshotSourceFixture(t, mgr, sourceID, sourceID, mgr.defaultHypervisor) + + meta, err := mgr.loadMetadata(sourceID) + require.NoError(t, err) + meta.GPUProfile = "NVIDIA L40S-2Q" + meta.GPUFramework = devices.VGPUFramework("future-framework") + meta.GPUDevicePath = "/sys/bus/pci/devices/0000:82:00.4" + meta.GPURetainedForCleanup = true + require.NoError(t, mgr.saveMetadata(meta)) + + // The delete-only retention stub has no boot configuration, so a snapshot + // of it could never be restored or forked into a bootable instance. + _, err = mgr.CreateSnapshot(ctx, sourceID, CreateSnapshotRequest{ + Kind: SnapshotKindStopped, + Name: "snapshot-vgpu-retention", + }) + require.ErrorIs(t, err, ErrInvalidState) + require.ErrorContains(t, err, "delete it to release the assignment") +} + func TestRestoreSnapshotDoesNotResurrectStaleVGPUAssignment(t *testing.T) { mgr, _ := setupTestManager(t) ctx := context.Background() From ab20441d7b3bb0fc65c7f6e24e130d5019fe5d3f Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Fri, 14 Aug 2026 20:56:18 +0000 Subject: [PATCH 39/76] Adapt vGPU liveness guards to the identity struct resolver The claim guard and startup reconcile protection predate the HypervisorProcessIdentity struct and the removal of the standalone identity-exists helpers. Route both through resolveLiveHypervisorPID: the claim guard keeps failing closed on unresolvable ownership, and reconcile protection gets a fail-open HypervisorMayBeAlive wrapper so unresolvable ownership still protects the device. --- cmd/api/main.go | 2 +- cmd/api/main_test.go | 4 +-- lib/instances/lifecycle_noop_test.go | 22 ++++++++--------- lib/instances/process_identity.go | 10 ++++++++ lib/instances/process_identity_linux_test.go | 12 ++++----- lib/instances/vgpu.go | 2 +- lib/instances/vgpu_test.go | 26 ++++++++++---------- 7 files changed, 44 insertions(+), 34 deletions(-) diff --git a/cmd/api/main.go b/cmd/api/main.go index f38a6a7c3..71e147ecf 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -196,7 +196,7 @@ func liveInstanceVGPUDevicePaths(ctx context.Context, instanceManager instances. if inst.GPUDevicePath == "" { continue } - if inst.HypervisorPID != nil && instances.HypervisorProcessIdentityExists(*inst.HypervisorPID, inst.HypervisorStartTime, inst.HypervisorBootID, inst.SocketPath) { + if inst.HypervisorPID != nil && instances.HypervisorMayBeAlive(inst.HypervisorProcessIdentity, inst.SocketPath) { protected[inst.GPUDevicePath] = struct{}{} continue } diff --git a/cmd/api/main_test.go b/cmd/api/main_test.go index 217e9db8d..85c132814 100644 --- a/cmd/api/main_test.go +++ b/cmd/api/main_test.go @@ -362,8 +362,8 @@ func TestLiveInstanceVGPUDevicePathsBoundsStartupProtection(t *testing.T) { {StoredMetadata: instances.StoredMetadata{Id: "booting", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", GPUAssignedAt: &recent}}, {StoredMetadata: instances.StoredMetadata{Id: "orphaned", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.5", GPUAssignedAt: &stale}}, {StoredMetadata: instances.StoredMetadata{Id: "legacy", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.6"}}, - {StoredMetadata: instances.StoredMetadata{Id: "dead", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.7", HypervisorPID: &deadPID}}, - {StoredMetadata: instances.StoredMetadata{Id: "stale-pid-booting", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.8", HypervisorPID: &deadPID, GPUAssignedAt: &recent}}, + {StoredMetadata: instances.StoredMetadata{Id: "dead", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.7", HypervisorProcessIdentity: instances.HypervisorProcessIdentity{HypervisorPID: &deadPID}}}, + {StoredMetadata: instances.StoredMetadata{Id: "stale-pid-booting", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.8", HypervisorProcessIdentity: instances.HypervisorProcessIdentity{HypervisorPID: &deadPID}, GPUAssignedAt: &recent}}, }} protected, retryAfter, err := liveInstanceVGPUDevicePaths(context.Background(), manager) diff --git a/lib/instances/lifecycle_noop_test.go b/lib/instances/lifecycle_noop_test.go index e440e7e4a..f5f495988 100644 --- a/lib/instances/lifecycle_noop_test.go +++ b/lib/instances/lifecycle_noop_test.go @@ -258,17 +258,17 @@ func TestDeleteDropsStaleVGPUClaimedByLiveInstance(t *testing.T) { require.NoError(t, err) defer listener.Close() require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ - Id: claimantID, - Name: claimantID, - Image: "test-image", - CreatedAt: now, - HypervisorType: lifecycleNoopHypervisorType, - HypervisorPID: &pid, - SocketPath: socketPath, - DataDir: m.paths.InstanceDir(claimantID), - GPUProfile: "NVIDIA L40S-2Q", - GPUFramework: devices.VGPUFrameworkVendorVFIO, - GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", + Id: claimantID, + Name: claimantID, + Image: "test-image", + CreatedAt: now, + HypervisorType: lifecycleNoopHypervisorType, + HypervisorProcessIdentity: HypervisorProcessIdentity{HypervisorPID: &pid}, + SocketPath: socketPath, + DataDir: m.paths.InstanceDir(claimantID), + GPUProfile: "NVIDIA L40S-2Q", + GPUFramework: devices.VGPUFrameworkVendorVFIO, + GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", }})) require.NoError(t, m.DeleteInstance(context.Background(), id)) diff --git a/lib/instances/process_identity.go b/lib/instances/process_identity.go index 1d885d0b5..be645e111 100644 --- a/lib/instances/process_identity.go +++ b/lib/instances/process_identity.go @@ -180,6 +180,16 @@ func classifyResolvedHypervisorOwner(socketPath string, stored, resolved int, er return 0, fmt.Errorf("cannot confirm ownership of socket %s: %w", socketPath, err) } +// HypervisorMayBeAlive reports whether the recorded hypervisor process may +// still be running. It fails open: when ownership cannot be resolved it +// returns true, which is the safe direction for its callers (reconcile +// protection and claim checks, where true means "protect"). Do not use it to +// authorize teardown. +func HypervisorMayBeAlive(id HypervisorProcessIdentity, socketPath string) bool { + pid, err := resolveLiveHypervisorPID(id, socketPath) + return err != nil || pid > 0 +} + // ProcessExists reports whether pid belongs to a live, non-zombie process. func ProcessExists(pid int) bool { if pid <= 0 { diff --git a/lib/instances/process_identity_linux_test.go b/lib/instances/process_identity_linux_test.go index d5bd973d9..7ad259c41 100644 --- a/lib/instances/process_identity_linux_test.go +++ b/lib/instances/process_identity_linux_test.go @@ -512,7 +512,7 @@ func TestRefreshHypervisorPIDResolvesSocketOwnerWhenStoredPIDIsDead(t *testing.T func TestVGPUAssignmentClaimedByLiveInstanceProtectsReusedPIDClaim(t *testing.T) { socketPath := filepath.Join(t.TempDir(), "test.sock") - owner := exec.Command(os.Args[0], "-test.run=^TestHypervisorProcessExistsWithReboundSocketPathHelper$") + owner := exec.Command(os.Args[0], "-test.run=^TestSocketListenerHelper$") owner.Env = append(os.Environ(), "HYPERVISOR_SOCKET_HELPER=1", "HYPERVISOR_SOCKET_PATH="+socketPath) stdin, err := owner.StdinPipe() require.NoError(t, err) @@ -539,11 +539,11 @@ func TestVGPUAssignmentClaimedByLiveInstanceProtectsReusedPIDClaim(t *testing.T) stalePID := stale.Process.Pid require.NoError(t, m.ensureDirectories("live-claimant")) require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ - Id: "live-claimant", - GPUFramework: devices.VGPUFrameworkVendorVFIO, - GPUDevicePath: devicePath, - HypervisorPID: &stalePID, - SocketPath: socketPath, + Id: "live-claimant", + GPUFramework: devices.VGPUFrameworkVendorVFIO, + GPUDevicePath: devicePath, + HypervisorProcessIdentity: HypervisorProcessIdentity{HypervisorPID: &stalePID}, + SocketPath: socketPath, }})) claimed, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "other-instance", devicePath) diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index 2f9cfa7d1..54a0bf9c6 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -201,7 +201,7 @@ func (m *manager) vgpuAssignmentClaimedByLiveInstance(ctx context.Context, exclu } return false, fmt.Errorf("cannot confirm liveness of recent vGPU claimant %s on %s: no persisted hypervisor PID", id, devicePath) } - pid, err := resolveLiveHypervisorPID(stored.HypervisorPID, stored.HypervisorStartTime, stored.HypervisorBootID, stored.SocketPath) + pid, err := resolveLiveHypervisorPID(stored.HypervisorProcessIdentity, stored.SocketPath) if err != nil { return false, fmt.Errorf("cannot confirm liveness of vGPU claimant %s on %s: %w", id, devicePath, err) } diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index a6fe4e49a..1d0e4e3bf 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -483,10 +483,10 @@ func TestVGPUAssignmentClaimedByLiveInstanceNormalizesLegacyMdevPath(t *testing. require.NoError(t, m.ensureDirectories("legacy-claimant")) pid := os.Getpid() require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ - Id: "legacy-claimant", - Name: "legacy-claimant", - GPUMdevUUID: "legacy-uuid", - HypervisorPID: &pid, + Id: "legacy-claimant", + Name: "legacy-claimant", + GPUMdevUUID: "legacy-uuid", + HypervisorProcessIdentity: HypervisorProcessIdentity{HypervisorPID: &pid}, }})) claimed, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "other-instance", "/sys/bus/mdev/devices/legacy-uuid") @@ -538,11 +538,11 @@ func TestVGPUAssignmentClaimedByLiveInstanceGracesRecentDeadPIDClaim(t *testing. require.False(t, ProcessExists(deadPID)) assignedAt := time.Now().UTC() require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ - Id: claimantID, - HypervisorPID: &deadPID, - GPUFramework: devices.VGPUFrameworkVendorVFIO, - GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", - GPUAssignedAt: &assignedAt, + Id: claimantID, + HypervisorProcessIdentity: HypervisorProcessIdentity{HypervisorPID: &deadPID}, + GPUFramework: devices.VGPUFrameworkVendorVFIO, + GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", + GPUAssignedAt: &assignedAt, }})) // Same bounded grace as startup reconcile: a recent claim whose PID is @@ -569,10 +569,10 @@ func TestVGPUAssignmentClaimedByLiveInstanceIgnoresDeadClaim(t *testing.T) { require.NoError(t, m.ensureDirectories("dead-claimant")) deadPID := 1 << 30 require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ - Id: "dead-claimant", - Name: "dead-claimant", - GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", - HypervisorPID: &deadPID, + Id: "dead-claimant", + Name: "dead-claimant", + GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", + HypervisorProcessIdentity: HypervisorProcessIdentity{HypervisorPID: &deadPID}, }})) claimed, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "other-instance", "/sys/bus/pci/devices/0000:82:00.4") From eb52f8c2d48af8afb0fb8304b8f2632d5917e8e8 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Mon, 17 Aug 2026 21:10:56 +0000 Subject: [PATCH 40/76] Deep-copy the phase tracker into the start rollback snapshot --- lib/instances/start.go | 1 + lib/instances/vgpu.go | 4 +--- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/instances/start.go b/lib/instances/start.go index 8914f2ae1..44dc50216 100644 --- a/lib/instances/start.go +++ b/lib/instances/start.go @@ -75,6 +75,7 @@ func (m *manager) startInstance( stored.HypervisorStartTime = 0 stored.HypervisorBootID = "" rollbackMeta := *meta + rollbackMeta.Phases = meta.Phases.Clone() // 2a. Clear stale exit info from previous run and apply command overrides stored.ExitCode = nil diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index 54a0bf9c6..d0cd9b226 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -92,9 +92,7 @@ func clearStoredVGPUDevice(stored *StoredMetadata) { // cleanup stack is LIFO, so cleanups registered after this one run before it // and this restore would clobber anything they persisted; it is safe only // while no such cleanup writes metadata and the instance lock serializes -// start. The snapshot is also a shallow copy (Phases shares its map), so it -// must be persisted before any Phases.Record on the live struct. Violating -// either invariant requires switching to targeted field restores. +// start. Violating that requires switching to targeted field restores. // // It reports whether the assignment was retained after a failed destroy and // whether that retention record was persisted, so start can surface the From 3f4c157949b0eaac44e53585e82416e446cbe851 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Tue, 18 Aug 2026 18:59:15 +0000 Subject: [PATCH 41/76] Retry orphaned vGPU releases in the background after delete MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A vGPU release during delete routinely fails when a GPU-busy VMM's kernel-side VFIO teardown outlives the force-kill wait. Delete's log-and-continue contract then deleted the metadata, stranding the VF until the next server restart and silently shrinking host GPU capacity. Hand the failed assignment to a bounded background retry (30s interval, 20 attempts) that re-runs the full release path — claim scan and destroy guards included — off the request path. The in-memory queue dedupes by device path; a restart abandons it and startup reconciliation sweeps the VF as before. --- lib/devices/GPU.md | 2 +- lib/instances/delete.go | 9 +- lib/instances/lifecycle_noop_test.go | 4 +- lib/instances/manager.go | 8 ++ lib/instances/vgpu_orphan.go | 78 ++++++++++++++ lib/instances/vgpu_orphan_test.go | 146 +++++++++++++++++++++++++++ 6 files changed, 241 insertions(+), 6 deletions(-) create mode 100644 lib/instances/vgpu_orphan.go create mode 100644 lib/instances/vgpu_orphan_test.go diff --git a/lib/devices/GPU.md b/lib/devices/GPU.md index 3a1504428..ea5764ee0 100644 --- a/lib/devices/GPU.md +++ b/lib/devices/GPU.md @@ -97,7 +97,7 @@ Instance Create → Assign profile to VF → Attach VF to VM → Instance Runnin Instance Stop/Delete → Release profile → VF available again ``` -Hypeman reconciles orphaned assignments on server restart while preserving devices held open by a running VMM. +Hypeman reconciles orphaned assignments on server restart while preserving devices held open by a running VMM. A release that fails during delete (typically because a GPU-busy VMM's kernel-side VFIO teardown outlives the force-kill wait) is retried in the background for up to ten minutes, so a completed delete does not strand the VF until the next restart. ### Hypervisor Support diff --git a/lib/instances/delete.go b/lib/instances/delete.go index 08781b977..f7646d64d 100644 --- a/lib/instances/delete.go +++ b/lib/instances/delete.go @@ -144,15 +144,18 @@ func (m *manager) deleteInstanceWithOptions( // or volume teardown. Release failure is logged and the delete continues, // matching the pre-refactor contract: the VMM is already confirmed dead, // the guards inside the release never destroy a device they cannot prove - // is unowned, and a skipped release is recovered by startup - // reconciliation. + // is unowned, and a skipped release is recovered by the background retry + // below or, after a restart, by startup reconciliation. hadVGPUAssignment := storedVGPUDevicePath(stored) != "" if hadVGPUAssignment { log.InfoContext(ctx, "destroying vGPU", "instance_id", id, "uuid", stored.GPUMdevUUID) } if err := m.releaseStoredVGPU(ctx, stored); err != nil { - // Log error but continue with cleanup. + // Log error but continue with cleanup. The metadata is about to be + // deleted, so hand the assignment to the background retry — otherwise + // the VF stays allocated until the next startup reconciliation. log.WarnContext(ctx, "failed to destroy vGPU, continuing with cleanup", "instance_id", id, "uuid", stored.GPUMdevUUID, "error", err) + m.scheduleOrphanedVGPURelease(ctx, *stored) } else if hadVGPUAssignment { if err := m.saveMetadata(meta); err != nil { log.WarnContext(ctx, "failed to save metadata after vGPU release", "instance_id", id, "error", err) diff --git a/lib/instances/lifecycle_noop_test.go b/lib/instances/lifecycle_noop_test.go index f5f495988..7868534a3 100644 --- a/lib/instances/lifecycle_noop_test.go +++ b/lib/instances/lifecycle_noop_test.go @@ -160,8 +160,8 @@ func TestDeleteContinuesWhenVGPUReleaseFails(t *testing.T) { require.NoError(t, m.saveMetadata(meta)) // A failed release is logged and the delete continues, matching the - // pre-refactor contract; the leaked assignment is recovered by startup - // reconciliation. + // pre-refactor contract; the leaked assignment is recovered by the + // background retry or startup reconciliation. require.NoError(t, m.DeleteInstance(context.Background(), id)) _, err = m.loadMetadata(id) diff --git a/lib/instances/manager.go b/lib/instances/manager.go index 1bb1e53ec..985eb29d7 100644 --- a/lib/instances/manager.go +++ b/lib/instances/manager.go @@ -213,6 +213,14 @@ type manager struct { // Periodic TAP garbage collection reconciler. tapGCOnce sync.Once + // vGPU assignments that survived a completed delete, keyed by device + // path, each with a background release retry in flight. + // orphanedVGPURetryDelay overrides the retry delay in tests; zero means + // the default. + orphanedVGPUMu sync.Mutex + orphanedVGPUs map[string]struct{} + orphanedVGPURetryDelay time.Duration + // Hypervisor support vmStarters map[hypervisor.Type]hypervisor.VMStarter defaultHypervisor hypervisor.Type // Default hypervisor type when not specified in request diff --git a/lib/instances/vgpu_orphan.go b/lib/instances/vgpu_orphan.go new file mode 100644 index 000000000..69a3545c5 --- /dev/null +++ b/lib/instances/vgpu_orphan.go @@ -0,0 +1,78 @@ +package instances + +import ( + "context" + "time" + + "github.com/kernel/hypeman/lib/logger" +) + +const ( + // orphanedVGPUReleaseMaxAttempts bounds the retry loop so a genuinely + // wedged VF degrades to one operator-actionable error instead of + // indefinite log churn. At the default interval this covers ten minutes, + // far beyond the seconds a dying VMM normally needs to finish kernel-side + // VFIO teardown. + orphanedVGPUReleaseMaxAttempts = 20 + defaultOrphanedVGPUReleaseRetryDelay = 30 * time.Second +) + +// scheduleOrphanedVGPURelease retries a vGPU release that failed during a +// completed delete, off the request path. Delete's log-and-continue contract +// is untouched — the caller already has its success — but without a retry the +// VF would stay allocated until the next startup reconciliation, silently +// shrinking host GPU capacity. A GPU-busy VMM routinely outlives delete's +// force-kill wait while the kernel finishes VFIO teardown, so this is the +// common case under load, not a tail case. +// +// Each attempt re-runs releaseStoredVGPU on a copy of the deleted instance's +// stored metadata, so the vendor VFIO claim scan and the destroy-side owner +// and open-handle guards apply on every retry exactly as they did on the +// original release. The queue is in-memory only: a restart abandons it and +// startup reconciliation sweeps the VF instead. +func (m *manager) scheduleOrphanedVGPURelease(ctx context.Context, stored StoredMetadata) { + path := storedVGPUDevicePath(&stored) + if path == "" { + return + } + m.orphanedVGPUMu.Lock() + if m.orphanedVGPUs == nil { + m.orphanedVGPUs = make(map[string]struct{}) + } + if _, pending := m.orphanedVGPUs[path]; pending { + m.orphanedVGPUMu.Unlock() + return + } + m.orphanedVGPUs[path] = struct{}{} + m.orphanedVGPUMu.Unlock() + + delay := m.orphanedVGPURetryDelay + if delay <= 0 { + delay = defaultOrphanedVGPUReleaseRetryDelay + } + // The request context ends with the delete; keep its values for logging + // but detach from its cancellation. + go m.retryOrphanedVGPURelease(context.WithoutCancel(ctx), stored, path, delay) +} + +func (m *manager) retryOrphanedVGPURelease(ctx context.Context, stored StoredMetadata, path string, delay time.Duration) { + log := logger.FromContext(ctx) + defer func() { + m.orphanedVGPUMu.Lock() + delete(m.orphanedVGPUs, path) + m.orphanedVGPUMu.Unlock() + }() + for attempt := 1; attempt <= orphanedVGPUReleaseMaxAttempts; attempt++ { + time.Sleep(delay) + if err := m.releaseStoredVGPU(ctx, &stored); err != nil { + log.WarnContext(ctx, "orphaned vGPU release retry failed", + "instance_id", stored.Id, "device_path", path, "attempt", attempt, "error", err) + continue + } + log.InfoContext(ctx, "released orphaned vGPU after delete", + "instance_id", stored.Id, "device_path", path, "attempt", attempt) + return + } + log.ErrorContext(ctx, "giving up on orphaned vGPU release; VF stays allocated until startup reconciliation or manual remediation", + "instance_id", stored.Id, "device_path", path, "attempts", orphanedVGPUReleaseMaxAttempts) +} diff --git a/lib/instances/vgpu_orphan_test.go b/lib/instances/vgpu_orphan_test.go new file mode 100644 index 000000000..25a350064 --- /dev/null +++ b/lib/instances/vgpu_orphan_test.go @@ -0,0 +1,146 @@ +package instances + +import ( + "context" + "errors" + "sync/atomic" + "testing" + "time" + + "github.com/kernel/hypeman/lib/devices" + "github.com/kernel/hypeman/lib/paths" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func waitForOrphanQueueEmpty(t *testing.T, m *manager) { + t.Helper() + require.Eventually(t, func() bool { + m.orphanedVGPUMu.Lock() + defer m.orphanedVGPUMu.Unlock() + return len(m.orphanedVGPUs) == 0 + }, 5*time.Second, 5*time.Millisecond, "orphan retry should finish and clear its queue entry") +} + +func TestScheduleOrphanedVGPUReleaseRetriesUntilSuccess(t *testing.T) { + t.Parallel() + + var attempts atomic.Int32 + m := &manager{ + paths: paths.New(t.TempDir()), + orphanedVGPURetryDelay: time.Millisecond, + destroyVGPU: func(context.Context, devices.VGPUAssignment) error { + if attempts.Add(1) < 3 { + return errors.New("operation not permitted") + } + return nil + }, + } + m.scheduleOrphanedVGPURelease(context.Background(), StoredMetadata{ + Id: "deleted-instance", + GPUFramework: devices.VGPUFrameworkVendorVFIO, + GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", + }) + + waitForOrphanQueueEmpty(t, m) + assert.Equal(t, int32(3), attempts.Load(), "release should succeed on the third attempt and stop retrying") +} + +func TestScheduleOrphanedVGPUReleaseGivesUpAfterMaxAttempts(t *testing.T) { + t.Parallel() + + var attempts atomic.Int32 + m := &manager{ + paths: paths.New(t.TempDir()), + orphanedVGPURetryDelay: time.Millisecond, + destroyVGPU: func(context.Context, devices.VGPUAssignment) error { + attempts.Add(1) + return errors.New("vGPU destroy failed: 0xffffffff") + }, + } + m.scheduleOrphanedVGPURelease(context.Background(), StoredMetadata{ + Id: "deleted-instance", + GPUFramework: devices.VGPUFrameworkVendorVFIO, + GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", + }) + + waitForOrphanQueueEmpty(t, m) + assert.Equal(t, int32(orphanedVGPUReleaseMaxAttempts), attempts.Load(), + "a wedged VF should get exactly the bounded number of attempts") +} + +func TestScheduleOrphanedVGPUReleaseDeduplicatesByDevicePath(t *testing.T) { + t.Parallel() + + var attempts atomic.Int32 + release := make(chan struct{}) + m := &manager{ + paths: paths.New(t.TempDir()), + orphanedVGPURetryDelay: 20 * time.Millisecond, + destroyVGPU: func(context.Context, devices.VGPUAssignment) error { + attempts.Add(1) + <-release + return nil + }, + } + stored := StoredMetadata{ + Id: "deleted-instance", + GPUFramework: devices.VGPUFrameworkVendorVFIO, + GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", + } + m.scheduleOrphanedVGPURelease(context.Background(), stored) + m.scheduleOrphanedVGPURelease(context.Background(), stored) + + require.Eventually(t, func() bool { return attempts.Load() == 1 }, 5*time.Second, 5*time.Millisecond) + close(release) + waitForOrphanQueueEmpty(t, m) + assert.Equal(t, int32(1), attempts.Load(), "the second schedule for the same path must be dropped") +} + +func TestScheduleOrphanedVGPUReleaseIgnoresEmptyAssignment(t *testing.T) { + t.Parallel() + + m := &manager{paths: paths.New(t.TempDir())} + m.scheduleOrphanedVGPURelease(context.Background(), StoredMetadata{Id: "no-gpu"}) + + m.orphanedVGPUMu.Lock() + defer m.orphanedVGPUMu.Unlock() + assert.Empty(t, m.orphanedVGPUs) +} + +// TestOrphanedVGPUReleaseReappliesClaimScan pins that the background retry +// goes through releaseStoredVGPU, not a raw destroy: a live claimant found by +// the vendor VFIO claim scan must keep blocking the release on every retry. +func TestOrphanedVGPUReleaseReappliesClaimScan(t *testing.T) { + t.Parallel() + + var destroys atomic.Int32 + m := &manager{ + paths: paths.New(t.TempDir()), + orphanedVGPURetryDelay: time.Millisecond, + destroyVGPU: func(context.Context, devices.VGPUAssignment) error { + destroys.Add(1) + return nil + }, + } + // A claimant with a recent assignment and no persisted PID makes the scan + // fail closed, exactly like the synchronous release path. + require.NoError(t, m.ensureDirectories("mid-boot-claimant")) + assignedAt := time.Now() + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: "mid-boot-claimant", + Name: "mid-boot-claimant", + GPUFramework: devices.VGPUFrameworkVendorVFIO, + GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", + GPUAssignedAt: &assignedAt, + }})) + + m.scheduleOrphanedVGPURelease(context.Background(), StoredMetadata{ + Id: "deleted-instance", + GPUFramework: devices.VGPUFrameworkVendorVFIO, + GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", + }) + + waitForOrphanQueueEmpty(t, m) + assert.Zero(t, destroys.Load(), "no destroy may fire while the claim scan cannot clear the path") +} From 942a33d810ed556d5f2b6f89e29ff2cd990e19fd Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Tue, 18 Aug 2026 19:40:11 +0000 Subject: [PATCH 42/76] Bound orphan retries in delete tests and tighten comments The two delete-continues tests triggered the new orphan retry with the default 30s delay, leaving a goroutine running ~10 minutes past the test. Use a millisecond delay and drain the queue before returning. --- lib/instances/delete.go | 5 ++--- lib/instances/lifecycle_noop_test.go | 4 ++++ lib/instances/vgpu_orphan.go | 18 ++++++------------ 3 files changed, 12 insertions(+), 15 deletions(-) diff --git a/lib/instances/delete.go b/lib/instances/delete.go index f7646d64d..49eead69c 100644 --- a/lib/instances/delete.go +++ b/lib/instances/delete.go @@ -151,9 +151,8 @@ func (m *manager) deleteInstanceWithOptions( log.InfoContext(ctx, "destroying vGPU", "instance_id", id, "uuid", stored.GPUMdevUUID) } if err := m.releaseStoredVGPU(ctx, stored); err != nil { - // Log error but continue with cleanup. The metadata is about to be - // deleted, so hand the assignment to the background retry — otherwise - // the VF stays allocated until the next startup reconciliation. + // Log error but continue with cleanup; the background retry releases + // the VF once the metadata is gone. log.WarnContext(ctx, "failed to destroy vGPU, continuing with cleanup", "instance_id", id, "uuid", stored.GPUMdevUUID, "error", err) m.scheduleOrphanedVGPURelease(ctx, *stored) } else if hadVGPUAssignment { diff --git a/lib/instances/lifecycle_noop_test.go b/lib/instances/lifecycle_noop_test.go index 7868534a3..ebb8a8da1 100644 --- a/lib/instances/lifecycle_noop_test.go +++ b/lib/instances/lifecycle_noop_test.go @@ -152,6 +152,7 @@ func TestLifecycleNoopStandbyWithOptionsStillRejectsStandbyInstance(t *testing.T func TestDeleteContinuesWhenVGPUReleaseFails(t *testing.T) { m, id := newLifecycleNoopManagerWithInstance(t, StateStopped, time.Now().UTC()) + m.orphanedVGPURetryDelay = time.Millisecond meta, err := m.loadMetadata(id) require.NoError(t, err) meta.GPUProfile = "NVIDIA L40S-2Q" @@ -163,6 +164,7 @@ func TestDeleteContinuesWhenVGPUReleaseFails(t *testing.T) { // pre-refactor contract; the leaked assignment is recovered by the // background retry or startup reconciliation. require.NoError(t, m.DeleteInstance(context.Background(), id)) + waitForOrphanQueueEmpty(t, m) _, err = m.loadMetadata(id) require.Error(t, err, "instance data must be deleted despite the failed release") @@ -282,6 +284,7 @@ func TestDeleteDropsStaleVGPUClaimedByLiveInstance(t *testing.T) { func TestDeleteContinuesTeardownAfterFailedVGPURelease(t *testing.T) { m, id := newLifecycleNoopManagerWithInstance(t, StateStopped, time.Now().UTC()) + m.orphanedVGPURetryDelay = time.Millisecond deviceManager := &recordingDeviceManager{} m.deviceManager = deviceManager meta, err := m.loadMetadata(id) @@ -295,6 +298,7 @@ func TestDeleteContinuesTeardownAfterFailedVGPURelease(t *testing.T) { // The failed release must not block the rest of the teardown: devices // are detached and the instance is fully deleted. require.NoError(t, m.DeleteInstance(context.Background(), id)) + waitForOrphanQueueEmpty(t, m) assert.Equal(t, []string{"dev-1"}, deviceManager.detached) _, err = m.loadMetadata(id) diff --git a/lib/instances/vgpu_orphan.go b/lib/instances/vgpu_orphan.go index 69a3545c5..5f3fc2b38 100644 --- a/lib/instances/vgpu_orphan.go +++ b/lib/instances/vgpu_orphan.go @@ -18,18 +18,12 @@ const ( ) // scheduleOrphanedVGPURelease retries a vGPU release that failed during a -// completed delete, off the request path. Delete's log-and-continue contract -// is untouched — the caller already has its success — but without a retry the -// VF would stay allocated until the next startup reconciliation, silently -// shrinking host GPU capacity. A GPU-busy VMM routinely outlives delete's -// force-kill wait while the kernel finishes VFIO teardown, so this is the -// common case under load, not a tail case. -// -// Each attempt re-runs releaseStoredVGPU on a copy of the deleted instance's -// stored metadata, so the vendor VFIO claim scan and the destroy-side owner -// and open-handle guards apply on every retry exactly as they did on the -// original release. The queue is in-memory only: a restart abandons it and -// startup reconciliation sweeps the VF instead. +// completed delete, off the request path. A GPU-busy VMM routinely outlives +// delete's force-kill wait while the kernel finishes VFIO teardown, and once +// the metadata is deleted nothing else releases the VF until the next +// startup reconciliation. Each attempt re-runs releaseStoredVGPU, so the +// claim scan and destroy guards apply on every retry. The queue is in-memory +// only: a restart abandons it and startup reconciliation sweeps the VF. func (m *manager) scheduleOrphanedVGPURelease(ctx context.Context, stored StoredMetadata) { path := storedVGPUDevicePath(&stored) if path == "" { From 70de8d44c392644b5670b78ad86cdda2ede58956 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Wed, 19 Aug 2026 03:54:42 +0000 Subject: [PATCH 43/76] SIGTERM vGPU QEMU before SIGKILL during driver init A SIGKILL delivered to QEMU while the NVIDIA vGPU plugin is still initializing the VF wedges it near-deterministically: the guest driver loops on RmInitAdapter timeouts with no host-side signal, and only an SR-IOV cycle of the parent GPU recovers it. Voluntary QEMU exits run their VFIO teardown and are safe, as are hard kills after init. Start-failure cleanup and the force-kill fallback for initializing vGPU instances now send SIGTERM and wait a bounded grace before SIGKILL, and a hard kill inside the init window logs the affected device path. Clean creates, graceful stops, and running-instance deletes are unchanged. --- lib/devices/GPU.md | 14 +++- lib/hypervisor/qemu/process.go | 55 ++++++++++++- lib/hypervisor/qemu/process_test.go | 44 ++++++++++ lib/instances/delete.go | 11 +++ lib/instances/manager.go | 4 + lib/instances/process_identity.go | 14 ++++ lib/instances/process_identity_linux_test.go | 85 ++++++++++++++++++++ 7 files changed, 219 insertions(+), 8 deletions(-) diff --git a/lib/devices/GPU.md b/lib/devices/GPU.md index ea5764ee0..2841d51c6 100644 --- a/lib/devices/GPU.md +++ b/lib/devices/GPU.md @@ -288,10 +288,16 @@ every request, so one wedged VF presents as all vGPU instances failing while `/resources` reports full capacity. The wedge itself leaves no host-side log: no kernel error, no XID, no plugin -crash. In the observed case it followed a period of heavy attach/teardown -churn on the VF, including QEMU processes that exited within seconds of -opening the VFIO device (failed start attempts that were then retried), so -suspect any workload that repeatedly kills the VMM mid-device-init. +crash. The trigger is a SIGKILL delivered to QEMU while the vGPU plugin is +still initializing the VF (roughly the first seconds after process start): +a single hard kill in that window wedges the VF near-deterministically, +while QEMU processes that exit voluntarily — error exits, QMP quit, SIGTERM — +run their VFIO teardown and never wedge, and hard kills of fully-initialized +vGPU VMs are also safe. Hypeman therefore SIGTERMs a vGPU QEMU first and only +escalates to SIGKILL after a grace period, both in start-failure cleanup and +when force-killing an instance that is still initializing; a hard kill in the +init window logs `VF may wedge` with the device path. External SIGKILLs +(OOM killer, manual `kill -9`) can still trigger it. Confirm by assigning the same profile on a different VF: if that guest initializes, the VF is wedged, not the driver stack. Remediate by cycling diff --git a/lib/hypervisor/qemu/process.go b/lib/hypervisor/qemu/process.go index c02c5beef..e7f8c84cc 100644 --- a/lib/hypervisor/qemu/process.go +++ b/lib/hypervisor/qemu/process.go @@ -40,6 +40,11 @@ const ( // socketDialTimeout is timeout for individual socket connection attempts socketDialTimeout = 100 * time.Millisecond + // vfioTermGrace is how long start-failure cleanup waits for a + // VFIO-attached QEMU to exit on SIGTERM before SIGKILL. Only failed + // starts pay it. + vfioTermGrace = 10 * time.Second + // clientCreateTimeout is how long to retry QMP client creation after the // socket appears. Under high parallel load the socket can accept connections // slightly later than file creation/availability. @@ -225,8 +230,14 @@ func buildQMPArgs(socketPath string) []string { } type startedProcess struct { - pid int - socketPath string + pid int + socketPath string + // termGrace, when non-zero, makes cleanup send SIGTERM and wait this long + // before SIGKILL. Set for VFIO-attached processes: hard-killing QEMU while + // the NVIDIA vGPU plugin is initializing can silently wedge the VF until + // its parent GPU's SR-IOV is cycled, while a terminating QEMU runs its + // device teardown and leaves the VF reusable. + termGrace time.Duration waitDone chan error waitConsumed bool waitErr error @@ -277,14 +288,47 @@ func (p *startedProcess) wait() error { return err } +// waitFor waits up to d for the process to exit, returning whether it did. +func (p *startedProcess) waitFor(d time.Duration) bool { + if _, exited := p.checkExited(); exited { + return true + } + select { + case err := <-p.waitDone: + p.waitConsumed = true + p.waitErr = err + return true + case <-time.After(d): + return false + } +} + func (p *startedProcess) cleanup() { if _, exited := p.checkExited(); !exited { - _ = syscall.Kill(p.pid, syscall.SIGKILL) - _ = p.wait() + terminated := false + if p.termGrace > 0 { + if syscall.Kill(p.pid, syscall.SIGTERM) == nil { + terminated = p.waitFor(p.termGrace) + } + } + if !terminated { + _ = syscall.Kill(p.pid, syscall.SIGKILL) + _ = p.wait() + } } _ = os.Remove(p.socketPath) } +// hasVFIODevice reports whether the QEMU command line attaches a VFIO device. +func hasVFIODevice(args []string) bool { + for _, arg := range args { + if strings.Contains(arg, "vfio-pci") { + return true + } + } + return false +} + // startQEMUProcess handles the common QEMU process startup logic. // Returns the PID, hypervisor client, and a cleanup function. // The cleanup function must be called on error; call cleanup.Release() on success. @@ -358,6 +402,9 @@ func (s *Starter) startQEMUProcess(ctx context.Context, p *paths.Paths, version } pid := proc.pid + if hasVFIODevice(args) { + proc.termGrace = vfioTermGrace + } log.DebugContext(processCtx, "QEMU process started", "pid", pid, "duration_ms", time.Since(processStartTime).Milliseconds()) // Setup cleanup to kill, reap, and remove the socket if subsequent steps fail. diff --git a/lib/hypervisor/qemu/process_test.go b/lib/hypervisor/qemu/process_test.go index e8be0dda2..b334887e3 100644 --- a/lib/hypervisor/qemu/process_test.go +++ b/lib/hypervisor/qemu/process_test.go @@ -1,6 +1,7 @@ package qemu import ( + "bufio" "context" "errors" "os" @@ -409,3 +410,46 @@ func TestWaitForSocketOrExitReturnsEarlyWhenProcessDies(t *testing.T) { require.NotNil(t, cmd.ProcessState) assert.True(t, cmd.ProcessState.Exited()) } + +func TestCleanupSIGTERMsProcessWithTermGrace(t *testing.T) { + socketPath := filepath.Join(t.TempDir(), "qemu.sock") + markerPath := filepath.Join(t.TempDir(), "terminated") + + cmd := exec.Command("sh", "-c", "trap 'touch "+markerPath+"; exit 0' TERM; echo ready; sleep 30 & wait") + stdout, err := cmd.StdoutPipe() + require.NoError(t, err) + proc, err := startManagedProcess(cmd, socketPath) + require.NoError(t, err) + _, err = bufio.NewReader(stdout).ReadString('\n') + require.NoError(t, err) + proc.termGrace = 5 * time.Second + + proc.cleanup() + + assert.FileExists(t, markerPath, "process must get SIGTERM, not SIGKILL, when termGrace is set") + require.NoFileExists(t, socketPath) + require.NotNil(t, cmd.ProcessState) +} + +func TestCleanupEscalatesToSIGKILLAfterTermGrace(t *testing.T) { + socketPath := filepath.Join(t.TempDir(), "qemu.sock") + + cmd := exec.Command("sh", "-c", "trap '' TERM; echo ready; sleep 30 & wait") + stdout, err := cmd.StdoutPipe() + require.NoError(t, err) + proc, err := startManagedProcess(cmd, socketPath) + require.NoError(t, err) + _, err = bufio.NewReader(stdout).ReadString('\n') + require.NoError(t, err) + proc.termGrace = 50 * time.Millisecond + + proc.cleanup() + + assert.ErrorIs(t, syscall.Kill(proc.pid, 0), syscall.ESRCH, "SIGTERM-ignoring process must still be hard-killed") + require.NoFileExists(t, socketPath) +} + +func TestHasVFIODevice(t *testing.T) { + assert.True(t, hasVFIODevice([]string{"-device", "vfio-pci,sysfsdev=/sys/bus/pci/devices/0000:82:00.4"})) + assert.False(t, hasVFIODevice([]string{"-device", "virtio-balloon-pci,id=balloon0"})) +} diff --git a/lib/instances/delete.go b/lib/instances/delete.go index 49eead69c..c72729f21 100644 --- a/lib/instances/delete.go +++ b/lib/instances/delete.go @@ -241,6 +241,17 @@ func (m *manager) killHypervisor(ctx context.Context, inst *Instance) error { log.WarnContext(ctx, "stored hypervisor PID does not own the instance socket, killing the socket owner", "instance_id", inst.Id, "stored_pid", *inst.HypervisorPID, "owner_pid", pid) } + if inst.GPUProfile != "" && inst.State == StateInitializing { + // SIGKILL during vGPU driver init can silently wedge the VF until + // its parent GPU's SR-IOV is cycled, so ask the VMM to exit and + // run its VFIO teardown first. + if syscall.Kill(pid, syscall.SIGTERM) == nil && WaitForProcessExit(pid, m.vgpuTermGrace()) { + os.Remove(inst.SocketPath) + return nil + } + log.WarnContext(ctx, "vGPU hypervisor did not exit on SIGTERM during driver init; hard-killing, VF may wedge", + "instance_id", inst.Id, "device_path", inst.GPUDevicePath) + } log.DebugContext(ctx, "killing hypervisor process", "instance_id", inst.Id, "pid", pid) if err := killProcessAndWait(pid); err != nil { return err diff --git a/lib/instances/manager.go b/lib/instances/manager.go index 985eb29d7..c3709f53f 100644 --- a/lib/instances/manager.go +++ b/lib/instances/manager.go @@ -221,6 +221,10 @@ type manager struct { orphanedVGPUs map[string]struct{} orphanedVGPURetryDelay time.Duration + // vgpuInitTermGrace overrides killHypervisor's SIGTERM wait for vGPU + // instances still initializing; zero means the default. + vgpuInitTermGrace time.Duration + // Hypervisor support vmStarters map[hypervisor.Type]hypervisor.VMStarter defaultHypervisor hypervisor.Type // Default hypervisor type when not specified in request diff --git a/lib/instances/process_identity.go b/lib/instances/process_identity.go index be645e111..cca38e916 100644 --- a/lib/instances/process_identity.go +++ b/lib/instances/process_identity.go @@ -25,6 +25,20 @@ const linuxBootIDPath = "/proc/sys/kernel/random/boot_id" // does not unstick it, so the wait is short to keep stop and delete fast. const hypervisorSIGKILLWaitTimeout = 2 * time.Second +// defaultVGPUInitTermGrace is how long killHypervisor waits for a vGPU +// hypervisor still in driver init to exit on SIGTERM before SIGKILL. Only +// the force-kill fallback pays it, and only for initializing vGPU instances. +const defaultVGPUInitTermGrace = 10 * time.Second + +// vgpuTermGrace returns the SIGTERM wait used before hard-killing an +// initializing vGPU hypervisor. +func (m *manager) vgpuTermGrace() time.Duration { + if m.vgpuInitTermGrace > 0 { + return m.vgpuInitTermGrace + } + return defaultVGPUInitTermGrace +} + // killProcessAndWait SIGKILLs pid and waits for it to exit. A process that // survives the first wait gets its process group killed too (the hypervisor // may have spawned children in its own group) and a short grace period. An diff --git a/lib/instances/process_identity_linux_test.go b/lib/instances/process_identity_linux_test.go index 7ad259c41..2121d151b 100644 --- a/lib/instances/process_identity_linux_test.go +++ b/lib/instances/process_identity_linux_test.go @@ -691,3 +691,88 @@ func TestResolveRuntimeHypervisorPIDMintsIdentityOnlyWhenConfirmed(t *testing.T) assert.Empty(t, stored.HypervisorBootID, "a dead fallback must not mint the identity token") }) } + +// startTrapProcess starts a shell with the given TERM trap action (empty +// ignores the signal) and blocks until the trap is installed. It returns the +// PID and its boot-scoped identity. +func startTrapProcess(t *testing.T, trapAction string) (int, HypervisorProcessIdentity) { + t.Helper() + script := fmt.Sprintf("trap '%s' TERM; echo ready; sleep 30 & wait", trapAction) + process := exec.Command("sh", "-c", script) + stdout, err := process.StdoutPipe() + require.NoError(t, err) + require.NoError(t, process.Start()) + _, err = bufio.NewReader(stdout).ReadString('\n') + require.NoError(t, err) + waitDone := make(chan error, 1) + go func() { waitDone <- process.Wait() }() + t.Cleanup(func() { + _ = process.Process.Kill() + <-waitDone + }) + + pid := process.Process.Pid + startTime := processStartTime(pid) + require.NotZero(t, startTime) + return pid, HypervisorProcessIdentity{HypervisorPID: &pid, HypervisorStartTime: startTime, HypervisorBootID: hostBootID()} +} + +func TestKillHypervisorSIGTERMsInitializingVGPUHypervisor(t *testing.T) { + markerPath := filepath.Join(t.TempDir(), "terminated") + pid, identity := startTrapProcess(t, "touch "+markerPath+"; exit 0") + socketPath := filepath.Join(t.TempDir(), "missing.sock") + + m := &manager{} + require.NoError(t, m.killHypervisor(context.Background(), &Instance{ + State: StateInitializing, + StoredMetadata: StoredMetadata{ + Id: "kill-test", + GPUProfile: "NVIDIA L40S-1Q", + HypervisorProcessIdentity: identity, + SocketPath: socketPath, + }, + })) + + require.Eventually(t, func() bool { + return syscall.Kill(pid, 0) == syscall.ESRCH + }, 5*time.Second, 10*time.Millisecond) + assert.FileExists(t, markerPath, "hypervisor must be given SIGTERM, not SIGKILL, during vGPU driver init") +} + +func TestKillHypervisorEscalatesToSIGKILLWhenSIGTERMIgnored(t *testing.T) { + pid, identity := startTrapProcess(t, "") + socketPath := filepath.Join(t.TempDir(), "missing.sock") + + m := &manager{vgpuInitTermGrace: 50 * time.Millisecond} + require.NoError(t, m.killHypervisor(context.Background(), &Instance{ + State: StateInitializing, + StoredMetadata: StoredMetadata{ + Id: "kill-test", + GPUProfile: "NVIDIA L40S-1Q", + HypervisorProcessIdentity: identity, + SocketPath: socketPath, + }, + })) + + assert.ErrorIs(t, syscall.Kill(pid, 0), syscall.ESRCH, "SIGTERM-ignoring hypervisor must still be hard-killed") +} + +func TestKillHypervisorHardKillsVGPUHypervisorPastInit(t *testing.T) { + markerPath := filepath.Join(t.TempDir(), "terminated") + pid, identity := startTrapProcess(t, "touch "+markerPath+"; exit 0") + socketPath := filepath.Join(t.TempDir(), "missing.sock") + + m := &manager{} + require.NoError(t, m.killHypervisor(context.Background(), &Instance{ + State: StateRunning, + StoredMetadata: StoredMetadata{ + Id: "kill-test", + GPUProfile: "NVIDIA L40S-1Q", + HypervisorProcessIdentity: identity, + SocketPath: socketPath, + }, + })) + + assert.ErrorIs(t, syscall.Kill(pid, 0), syscall.ESRCH) + assert.NoFileExists(t, markerPath, "running vGPU hypervisors keep the direct SIGKILL path") +} From c64689dcba94232b7e39d465ce0b44844be0245a Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Wed, 19 Aug 2026 14:07:44 +0000 Subject: [PATCH 44/76] Apply vGPU SIGTERM grace on stop's direct kill paths shutdownHypervisor (stop) could still SIGKILL an initializing vGPU QEMU directly: on QMP connect failure, on graceful-quit timeout, and when the hypervisor lacks graceful shutdown, bypassing the grace killHypervisor applies. Extract the SIGTERM-then-SIGKILL escalation into terminateThenKill and use it at all four force-kill sites. --- lib/hypervisor/qemu/process_test.go | 5 ----- lib/instances/delete.go | 13 +------------ lib/instances/manager.go | 2 +- lib/instances/process_identity.go | 21 +++++++++++++++++++-- lib/instances/standby.go | 6 +++--- 5 files changed, 24 insertions(+), 23 deletions(-) diff --git a/lib/hypervisor/qemu/process_test.go b/lib/hypervisor/qemu/process_test.go index b334887e3..2bdf27d4f 100644 --- a/lib/hypervisor/qemu/process_test.go +++ b/lib/hypervisor/qemu/process_test.go @@ -448,8 +448,3 @@ func TestCleanupEscalatesToSIGKILLAfterTermGrace(t *testing.T) { assert.ErrorIs(t, syscall.Kill(proc.pid, 0), syscall.ESRCH, "SIGTERM-ignoring process must still be hard-killed") require.NoFileExists(t, socketPath) } - -func TestHasVFIODevice(t *testing.T) { - assert.True(t, hasVFIODevice([]string{"-device", "vfio-pci,sysfsdev=/sys/bus/pci/devices/0000:82:00.4"})) - assert.False(t, hasVFIODevice([]string{"-device", "virtio-balloon-pci,id=balloon0"})) -} diff --git a/lib/instances/delete.go b/lib/instances/delete.go index c72729f21..2f90a8552 100644 --- a/lib/instances/delete.go +++ b/lib/instances/delete.go @@ -241,19 +241,8 @@ func (m *manager) killHypervisor(ctx context.Context, inst *Instance) error { log.WarnContext(ctx, "stored hypervisor PID does not own the instance socket, killing the socket owner", "instance_id", inst.Id, "stored_pid", *inst.HypervisorPID, "owner_pid", pid) } - if inst.GPUProfile != "" && inst.State == StateInitializing { - // SIGKILL during vGPU driver init can silently wedge the VF until - // its parent GPU's SR-IOV is cycled, so ask the VMM to exit and - // run its VFIO teardown first. - if syscall.Kill(pid, syscall.SIGTERM) == nil && WaitForProcessExit(pid, m.vgpuTermGrace()) { - os.Remove(inst.SocketPath) - return nil - } - log.WarnContext(ctx, "vGPU hypervisor did not exit on SIGTERM during driver init; hard-killing, VF may wedge", - "instance_id", inst.Id, "device_path", inst.GPUDevicePath) - } log.DebugContext(ctx, "killing hypervisor process", "instance_id", inst.Id, "pid", pid) - if err := killProcessAndWait(pid); err != nil { + if err := m.terminateThenKill(ctx, inst, pid); err != nil { return err } } diff --git a/lib/instances/manager.go b/lib/instances/manager.go index c3709f53f..846b73ce5 100644 --- a/lib/instances/manager.go +++ b/lib/instances/manager.go @@ -221,7 +221,7 @@ type manager struct { orphanedVGPUs map[string]struct{} orphanedVGPURetryDelay time.Duration - // vgpuInitTermGrace overrides killHypervisor's SIGTERM wait for vGPU + // vgpuInitTermGrace overrides terminateThenKill's SIGTERM wait for vGPU // instances still initializing; zero means the default. vgpuInitTermGrace time.Duration diff --git a/lib/instances/process_identity.go b/lib/instances/process_identity.go index cca38e916..9bfe2cdaa 100644 --- a/lib/instances/process_identity.go +++ b/lib/instances/process_identity.go @@ -1,6 +1,7 @@ package instances import ( + "context" "errors" "fmt" "os" @@ -13,6 +14,7 @@ import ( "time" "github.com/kernel/hypeman/lib/hypervisor" + "github.com/kernel/hypeman/lib/logger" ) // linuxBootIDPath is the kernel-provided boot ID used to scope process @@ -25,9 +27,9 @@ const linuxBootIDPath = "/proc/sys/kernel/random/boot_id" // does not unstick it, so the wait is short to keep stop and delete fast. const hypervisorSIGKILLWaitTimeout = 2 * time.Second -// defaultVGPUInitTermGrace is how long killHypervisor waits for a vGPU +// defaultVGPUInitTermGrace is how long terminateThenKill waits for a vGPU // hypervisor still in driver init to exit on SIGTERM before SIGKILL. Only -// the force-kill fallback pays it, and only for initializing vGPU instances. +// force-kill paths pay it, and only for initializing vGPU instances. const defaultVGPUInitTermGrace = 10 * time.Second // vgpuTermGrace returns the SIGTERM wait used before hard-killing an @@ -39,6 +41,21 @@ func (m *manager) vgpuTermGrace() time.Duration { return defaultVGPUInitTermGrace } +// terminateThenKill hard-kills the hypervisor process, first giving a vGPU +// instance still in driver init a SIGTERM grace: SIGKILL in that window can +// silently wedge the VF until its parent GPU's SR-IOV is cycled (see +// lib/devices/GPU.md), while a terminating QEMU runs its VFIO teardown. +func (m *manager) terminateThenKill(ctx context.Context, inst *Instance, pid int) error { + if inst.GPUProfile != "" && inst.State == StateInitializing { + if syscall.Kill(pid, syscall.SIGTERM) == nil && WaitForProcessExit(pid, m.vgpuTermGrace()) { + return nil + } + logger.FromContext(ctx).WarnContext(ctx, "vGPU hypervisor did not exit on SIGTERM during driver init; hard-killing, VF may wedge", + "instance_id", inst.Id, "device_path", inst.GPUDevicePath) + } + return killProcessAndWait(pid) +} + // killProcessAndWait SIGKILLs pid and waits for it to exit. A process that // survives the first wait gets its process group killed too (the hypervisor // may have spawned children in its own group) and a short grace period. An diff --git a/lib/instances/standby.go b/lib/instances/standby.go index 4c6fd7d33..5c6e0173e 100644 --- a/lib/instances/standby.go +++ b/lib/instances/standby.go @@ -376,7 +376,7 @@ func (m *manager) shutdownHypervisor(ctx context.Context, inst *Instance) error // alive; teardown is committed, so kill it rather than report a // completed shutdown for a VMM that is still running. log.WarnContext(ctx, "could not connect to hypervisor, force killing resolved owner", "instance_id", inst.Id, "pid", pid, "error", err) - if err := killProcessAndWait(pid); err != nil { + if err := m.terminateThenKill(ctx, inst, pid); err != nil { return err } } @@ -405,13 +405,13 @@ func (m *manager) shutdownHypervisor(ctx context.Context, inst *Instance) error log.DebugContext(ctx, "hypervisor shutdown gracefully", "instance_id", inst.Id, "pid", pid) } else { log.WarnContext(ctx, "hypervisor did not exit gracefully in time, force killing process", "instance_id", inst.Id, "pid", pid) - if err := killProcessAndWait(pid); err != nil { + if err := m.terminateThenKill(ctx, inst, pid); err != nil { return err } } } else { log.DebugContext(ctx, "skipping graceful exit wait; force killing hypervisor process", "instance_id", inst.Id, "pid", pid) - if err := killProcessAndWait(pid); err != nil { + if err := m.terminateThenKill(ctx, inst, pid); err != nil { return err } } From 53ff4252b5d503825a5527623e221495689bba56 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Wed, 19 Aug 2026 14:38:06 +0000 Subject: [PATCH 45/76] Lower vGPU SIGTERM grace to 5s Observed mid-init VFIO teardown completes in 1-2s, so 5s keeps 2-3x margin while halving the worst-case delay for a SIGTERM-ignoring process. --- lib/hypervisor/qemu/process.go | 5 +++-- lib/instances/process_identity.go | 6 ++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/lib/hypervisor/qemu/process.go b/lib/hypervisor/qemu/process.go index e7f8c84cc..cddd51824 100644 --- a/lib/hypervisor/qemu/process.go +++ b/lib/hypervisor/qemu/process.go @@ -42,8 +42,9 @@ const ( // vfioTermGrace is how long start-failure cleanup waits for a // VFIO-attached QEMU to exit on SIGTERM before SIGKILL. Only failed - // starts pay it. - vfioTermGrace = 10 * time.Second + // starts pay it, and only when the process ignores SIGTERM; observed + // mid-init VFIO teardown takes 1-2s. + vfioTermGrace = 5 * time.Second // clientCreateTimeout is how long to retry QMP client creation after the // socket appears. Under high parallel load the socket can accept connections diff --git a/lib/instances/process_identity.go b/lib/instances/process_identity.go index 9bfe2cdaa..2f68b6c92 100644 --- a/lib/instances/process_identity.go +++ b/lib/instances/process_identity.go @@ -29,8 +29,10 @@ const hypervisorSIGKILLWaitTimeout = 2 * time.Second // defaultVGPUInitTermGrace is how long terminateThenKill waits for a vGPU // hypervisor still in driver init to exit on SIGTERM before SIGKILL. Only -// force-kill paths pay it, and only for initializing vGPU instances. -const defaultVGPUInitTermGrace = 10 * time.Second +// force-kill paths pay it, only for initializing vGPU instances, and only +// when the process ignores SIGTERM; observed mid-init VFIO teardown takes +// 1-2s. +const defaultVGPUInitTermGrace = 5 * time.Second // vgpuTermGrace returns the SIGTERM wait used before hard-killing an // initializing vGPU hypervisor. From c9e7ece535dcec3b79ba691d6af2324d8b515a59 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Thu, 20 Aug 2026 18:52:27 +0000 Subject: [PATCH 46/76] Apply vGPU SIGTERM grace in every instance state Running reports true ~4s before the guest driver finishes initializing and no host-side signal observes that boundary, so gating the SIGTERM grace on StateInitializing left a window where a failed stop or delete could SIGKILL QEMU mid-driver-init and wedge the VF. Post-init the SIGTERM is proven harmless and costs the grace only when the process ignores it. --- lib/devices/GPU.md | 7 ++--- lib/instances/process_identity.go | 27 +++++++++++--------- lib/instances/process_identity_linux_test.go | 25 ++++++++++++++++-- 3 files changed, 42 insertions(+), 17 deletions(-) diff --git a/lib/devices/GPU.md b/lib/devices/GPU.md index 2841d51c6..ff11c5cde 100644 --- a/lib/devices/GPU.md +++ b/lib/devices/GPU.md @@ -295,9 +295,10 @@ while QEMU processes that exit voluntarily — error exits, QMP quit, SIGTERM run their VFIO teardown and never wedge, and hard kills of fully-initialized vGPU VMs are also safe. Hypeman therefore SIGTERMs a vGPU QEMU first and only escalates to SIGKILL after a grace period, both in start-failure cleanup and -when force-killing an instance that is still initializing; a hard kill in the -init window logs `VF may wedge` with the device path. External SIGKILLs -(OOM killer, manual `kill -9`) can still trigger it. +when force-killing any vGPU instance (the instance reports Running seconds +before driver init completes, so no state reliably marks the window); a hard +kill after an ignored SIGTERM logs `VF may wedge` with the device path. +External SIGKILLs (OOM killer, manual `kill -9`) can still trigger it. Confirm by assigning the same profile on a different VF: if that guest initializes, the VF is wedged, not the driver stack. Remediate by cycling diff --git a/lib/instances/process_identity.go b/lib/instances/process_identity.go index 2f68b6c92..0801aca6c 100644 --- a/lib/instances/process_identity.go +++ b/lib/instances/process_identity.go @@ -28,14 +28,13 @@ const linuxBootIDPath = "/proc/sys/kernel/random/boot_id" const hypervisorSIGKILLWaitTimeout = 2 * time.Second // defaultVGPUInitTermGrace is how long terminateThenKill waits for a vGPU -// hypervisor still in driver init to exit on SIGTERM before SIGKILL. Only -// force-kill paths pay it, only for initializing vGPU instances, and only -// when the process ignores SIGTERM; observed mid-init VFIO teardown takes -// 1-2s. +// hypervisor to exit on SIGTERM before SIGKILL. Only force-kill paths pay it, +// only for vGPU instances, and only when the process ignores SIGTERM; +// observed mid-init VFIO teardown takes 1-2s. const defaultVGPUInitTermGrace = 5 * time.Second -// vgpuTermGrace returns the SIGTERM wait used before hard-killing an -// initializing vGPU hypervisor. +// vgpuTermGrace returns the SIGTERM wait used before hard-killing a vGPU +// hypervisor. func (m *manager) vgpuTermGrace() time.Duration { if m.vgpuInitTermGrace > 0 { return m.vgpuInitTermGrace @@ -43,16 +42,20 @@ func (m *manager) vgpuTermGrace() time.Duration { return defaultVGPUInitTermGrace } -// terminateThenKill hard-kills the hypervisor process, first giving a vGPU -// instance still in driver init a SIGTERM grace: SIGKILL in that window can -// silently wedge the VF until its parent GPU's SR-IOV is cycled (see -// lib/devices/GPU.md), while a terminating QEMU runs its VFIO teardown. +// terminateThenKill hard-kills the hypervisor process, first giving any vGPU +// instance a SIGTERM grace: SIGKILL during guest driver init can silently +// wedge the VF until its parent GPU's SR-IOV is cycled (see +// lib/devices/GPU.md), while a terminating QEMU runs its VFIO teardown. The +// grace applies in every state, not just Initializing, because the instance +// reports Running seconds before the guest driver finishes initializing and +// nothing host-side observes that boundary; post-init the SIGTERM is proven +// harmless and costs the grace only when the process ignores it. func (m *manager) terminateThenKill(ctx context.Context, inst *Instance, pid int) error { - if inst.GPUProfile != "" && inst.State == StateInitializing { + if inst.GPUProfile != "" { if syscall.Kill(pid, syscall.SIGTERM) == nil && WaitForProcessExit(pid, m.vgpuTermGrace()) { return nil } - logger.FromContext(ctx).WarnContext(ctx, "vGPU hypervisor did not exit on SIGTERM during driver init; hard-killing, VF may wedge", + logger.FromContext(ctx).WarnContext(ctx, "vGPU hypervisor did not exit on SIGTERM; hard-killing, VF may wedge if the guest driver was initializing", "instance_id", inst.Id, "device_path", inst.GPUDevicePath) } return killProcessAndWait(pid) diff --git a/lib/instances/process_identity_linux_test.go b/lib/instances/process_identity_linux_test.go index 2121d151b..e090b9a98 100644 --- a/lib/instances/process_identity_linux_test.go +++ b/lib/instances/process_identity_linux_test.go @@ -757,7 +757,7 @@ func TestKillHypervisorEscalatesToSIGKILLWhenSIGTERMIgnored(t *testing.T) { assert.ErrorIs(t, syscall.Kill(pid, 0), syscall.ESRCH, "SIGTERM-ignoring hypervisor must still be hard-killed") } -func TestKillHypervisorHardKillsVGPUHypervisorPastInit(t *testing.T) { +func TestKillHypervisorSIGTERMsRunningVGPUHypervisor(t *testing.T) { markerPath := filepath.Join(t.TempDir(), "terminated") pid, identity := startTrapProcess(t, "touch "+markerPath+"; exit 0") socketPath := filepath.Join(t.TempDir(), "missing.sock") @@ -773,6 +773,27 @@ func TestKillHypervisorHardKillsVGPUHypervisorPastInit(t *testing.T) { }, })) + require.Eventually(t, func() bool { + return syscall.Kill(pid, 0) == syscall.ESRCH + }, 5*time.Second, 10*time.Millisecond) + assert.FileExists(t, markerPath, "Running reports true before guest driver init completes, so vGPU hypervisors get SIGTERM in every state") +} + +func TestKillHypervisorHardKillsNonVGPUHypervisor(t *testing.T) { + markerPath := filepath.Join(t.TempDir(), "terminated") + pid, identity := startTrapProcess(t, "touch "+markerPath+"; exit 0") + socketPath := filepath.Join(t.TempDir(), "missing.sock") + + m := &manager{} + require.NoError(t, m.killHypervisor(context.Background(), &Instance{ + State: StateRunning, + StoredMetadata: StoredMetadata{ + Id: "kill-test", + HypervisorProcessIdentity: identity, + SocketPath: socketPath, + }, + })) + assert.ErrorIs(t, syscall.Kill(pid, 0), syscall.ESRCH) - assert.NoFileExists(t, markerPath, "running vGPU hypervisors keep the direct SIGKILL path") + assert.NoFileExists(t, markerPath, "non-vGPU hypervisors keep the direct SIGKILL path") } From 7948ce9b05528b93dc21843f0667a666a7888efe Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Thu, 20 Aug 2026 20:12:33 +0000 Subject: [PATCH 47/76] Harden the vGPU claim scan and retention-stub stop path A concurrent instance deletion between the claim scan's metadata listing and load turned ErrNotFound into a host-wide fail-closed release error, even though a vanished record cannot be a live claimant; skip it. Stop on a delete-only retention stub released its VF while leaving GPURetainedForCleanup set, so the stub's start/fork/snapshot errors kept claiming an assignment that no longer existed. Retention stubs now release only through delete, as documented. --- lib/instances/lifecycle_noop_test.go | 22 ++++++++++++++++++++++ lib/instances/vgpu.go | 11 +++++++++++ 2 files changed, 33 insertions(+) diff --git a/lib/instances/lifecycle_noop_test.go b/lib/instances/lifecycle_noop_test.go index ebb8a8da1..46f891379 100644 --- a/lib/instances/lifecycle_noop_test.go +++ b/lib/instances/lifecycle_noop_test.go @@ -367,6 +367,28 @@ func TestStopStoppedInstanceReleasesRetainedVGPU(t *testing.T) { assert.Empty(t, stored.GPUDevicePath) } +func TestStopStoppedInstanceLeavesRetentionStubForDelete(t *testing.T) { + m, id := newLifecycleNoopManagerWithInstance(t, StateStopped, time.Now().UTC()) + meta, err := m.loadMetadata(id) + require.NoError(t, err) + meta.GPUProfile = "NVIDIA L40S-2Q" + meta.GPUFramework = devices.VGPUFrameworkNone + meta.GPUDevicePath = "/sys/bus/pci/devices/0000:82:00.4" + meta.GPURetainedForCleanup = true + require.NoError(t, m.saveMetadata(meta)) + + inst, err := m.StopInstance(context.Background(), id) + require.NoError(t, err) + require.NotNil(t, inst) + + // Retention stubs are delete-only: releasing on stop would leave a stub + // whose start/fork/snapshot errors still claim a retained assignment. + stored, err := m.loadMetadata(id) + require.NoError(t, err) + assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", stored.GPUDevicePath) + assert.True(t, stored.GPURetainedForCleanup) +} + func TestStopStoppedInstanceVGPUReleaseFailureRemainsNoop(t *testing.T) { m, id := newLifecycleNoopManagerWithInstance(t, StateStopped, time.Now().UTC()) meta, err := m.loadMetadata(id) diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index d0cd9b226..6b707e94c 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -187,6 +187,11 @@ func (m *manager) vgpuAssignmentClaimedByLiveInstance(ctx context.Context, exclu } meta, err := m.loadMetadata(id) if err != nil { + if errors.Is(err, ErrNotFound) { + // Deleted between listing and load; a vanished record cannot + // be a live claimant. + continue + } return false, fmt.Errorf("load metadata for vGPU release check: instance %s: %w", id, err) } stored := &meta.StoredMetadata @@ -228,6 +233,12 @@ func (m *manager) releaseRetainedVGPULocked(ctx context.Context, id string) { return } stored := &meta.StoredMetadata + if stored.GPURetainedForCleanup { + // Delete-only retention stubs release through delete. Releasing here + // would leave a stub whose start/fork/snapshot errors still claim a + // retained assignment that no longer exists. + return + } if storedVGPUDevicePath(stored) == "" { return } From bb8614ab866d2dbc40a9e14de11c3ae0a16899eb Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Fri, 21 Aug 2026 15:10:59 +0000 Subject: [PATCH 48/76] Share one vGPU retention stub in create The two rollback paths carried near-identical retention-stub literals that would drift as fields are added. Build both from one helper, use nowUTC like the rest of the file, and drop the starter guard that is dead since create fails on a nil starter long before the vGPU block. --- lib/instances/create.go | 45 +++++++++++++++++------------------------ 1 file changed, 19 insertions(+), 26 deletions(-) diff --git a/lib/instances/create.go b/lib/instances/create.go index e7ab3b6ad..eefd0a7ce 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -309,24 +309,27 @@ func (m *manager) createInstance( // Handle vGPU profile request if req.GPU != nil && req.GPU.Profile != "" { - log.InfoContext(ctx, "creating vGPU", "instance_id", id, "profile", req.GPU.Profile) - gpuDevice, err = m.createVGPUDevice(ctx, req.GPU.Profile, id) - if err != nil { - stub := StoredMetadata{ + // Identity fields a retention record keeps when rollback cannot + // release the assignment, so it lists as a recognizable, deletable + // instance. Create has already failed on a nil starter by this point. + retentionStub := func() StoredMetadata { + return StoredMetadata{ Id: id, Name: req.Name, Image: req.Image, ResolvedImage: resolvedImageRef, Platform: imageInfo.Platform, - CreatedAt: time.Now(), + CreatedAt: m.nowUTC(), HypervisorType: hvType, HypervisorVersion: hvVersion, + SocketPath: m.paths.InstanceSocket(id, starter.SocketName()), DataDir: m.paths.InstanceDir(id), } - if starterErr == nil { - stub.SocketPath = m.paths.InstanceSocket(id, starter.SocketName()) - } - retainedVGPU = retainedVGPUFromCreateError(stub, m.nowUTC(), err) + } + log.InfoContext(ctx, "creating vGPU", "instance_id", id, "profile", req.GPU.Profile) + gpuDevice, err = m.createVGPUDevice(ctx, req.GPU.Profile, id) + if err != nil { + retainedVGPU = retainedVGPUFromCreateError(retentionStub(), m.nowUTC(), err) log.ErrorContext(ctx, "failed to create vGPU", "profile", req.GPU.Profile, "error", err) return nil, wrapCreateVGPUErr(req.GPU.Profile, err) } @@ -351,23 +354,13 @@ func (m *manager) createInstance( log.WarnContext(ctx, "failed to destroy vGPU on cleanup", "instance_id", id, "uuid", gpuDevice.MdevUUID, "error", err) retainedVGPU = stored if retainedVGPU == nil { - retainedVGPU = &StoredMetadata{ - Id: id, - Name: req.Name, - Image: req.Image, - ResolvedImage: resolvedImageRef, - Platform: imageInfo.Platform, - CreatedAt: time.Now(), - HypervisorType: hvType, - HypervisorVersion: hvVersion, - SocketPath: m.paths.InstanceSocket(id, starter.SocketName()), - DataDir: m.paths.InstanceDir(id), - GPUProfile: gpuDevice.ProfileName, - GPUFramework: gpuDevice.Framework, - GPUDevicePath: gpuDevice.SysfsPath, - GPUMdevUUID: gpuDevice.MdevUUID, - GPUAssignedAt: gpuAssignedAt, - } + stub := retentionStub() + stub.GPUProfile = gpuDevice.ProfileName + stub.GPUFramework = gpuDevice.Framework + stub.GPUDevicePath = gpuDevice.SysfsPath + stub.GPUMdevUUID = gpuDevice.MdevUUID + stub.GPUAssignedAt = gpuAssignedAt + retainedVGPU = &stub } } }) From 54c37b31aa3230f7a95140c69484f758a9aeca33 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Fri, 21 Aug 2026 16:31:49 +0000 Subject: [PATCH 49/76] Name the strict metadata listing variant Replace the listMetadataFilesWithStatErrors(bool) mode flag with listMetadataFiles / listMetadataFilesStrict so call sites say which failure semantics they rely on. --- lib/instances/storage.go | 15 ++++++++++++--- lib/instances/vgpu.go | 2 +- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/lib/instances/storage.go b/lib/instances/storage.go index dd932d41a..6a2354624 100644 --- a/lib/instances/storage.go +++ b/lib/instances/storage.go @@ -187,12 +187,21 @@ func removeAllWithRetry(path string, removeAll func(string) error, sleep func(ti } } -// listMetadataFiles returns paths to all instance metadata files. +// listMetadataFiles returns paths to all instance metadata files, skipping +// entries whose metadata cannot be statted. func (m *manager) listMetadataFiles() ([]string, error) { - return m.listMetadataFilesWithStatErrors(false) + return m.walkMetadataFiles(false) } -func (m *manager) listMetadataFilesWithStatErrors(failOnStatError bool) ([]string, error) { +// listMetadataFilesStrict returns paths to all instance metadata files, +// failing on any stat error other than absence. Fail-closed callers (the +// vGPU release claim scan and startup reconcile protection) use it so an +// unreadable instance is an error instead of silently missing. +func (m *manager) listMetadataFilesStrict() ([]string, error) { + return m.walkMetadataFiles(true) +} + +func (m *manager) walkMetadataFiles(failOnStatError bool) ([]string, error) { guestsDir := m.paths.GuestsDir() // Ensure guests directory exists diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index 6b707e94c..b38fb5225 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -176,7 +176,7 @@ func (m *manager) releaseStoredVGPU(ctx context.Context, stored *StoredMetadata) // assignment without a PID, or unverifiable process ownership returns an error // so the requester retains its assignment for a later retry. func (m *manager) vgpuAssignmentClaimedByLiveInstance(ctx context.Context, excludeID, devicePath string) (bool, error) { - files, err := m.listMetadataFilesWithStatErrors(true) + files, err := m.listMetadataFilesStrict() if err != nil { return false, fmt.Errorf("list instances for vGPU release check: %w", err) } From b86d1da2e85bf95167505bb826b8c735a64d53c1 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Fri, 21 Aug 2026 16:31:49 +0000 Subject: [PATCH 50/76] Skip instances deleted mid-listing in reconcile protection ListInstancesForReconcile failed hard when an instance was deleted between the metadata listing and its load. The startup call runs before the API serves, but the grace-period retry fires while deletes are in flight; one racing delete errored the whole list, which zeroed the retry and left vendor VFIO reconciliation disabled until the next restart. Skip ErrNotFound like the release claim scan does: a vanished record cannot claim a VF. --- lib/instances/manager.go | 10 ++++++++- lib/instances/query_test.go | 41 +++++++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 1 deletion(-) diff --git a/lib/instances/manager.go b/lib/instances/manager.go index 846b73ce5..54e9b00d6 100644 --- a/lib/instances/manager.go +++ b/lib/instances/manager.go @@ -2,6 +2,7 @@ package instances import ( "context" + "errors" "fmt" "os" "path/filepath" @@ -755,7 +756,7 @@ func (m *manager) DefaultHypervisor() hypervisor.Type { // needs raw metadata fields, and hydration would query the hypervisor of // every instance on the host before the API serves. func (m *manager) ListInstancesForReconcile(ctx context.Context) ([]Instance, error) { - files, err := m.listMetadataFilesWithStatErrors(true) + files, err := m.listMetadataFilesStrict() if err != nil { return nil, err } @@ -764,6 +765,13 @@ func (m *manager) ListInstancesForReconcile(ctx context.Context) ([]Instance, er id := filepath.Base(filepath.Dir(file)) meta, err := m.loadMetadata(id) if err != nil { + if errors.Is(err, ErrNotFound) { + // Deleted between listing and load; a vanished record cannot + // claim a VF. Failing here instead would zero the grace-period + // retry and disable the vendor VFIO sweep whenever it races a + // concurrent delete. + continue + } return nil, fmt.Errorf("load metadata for instance %s: %w", id, err) } result = append(result, Instance{StoredMetadata: meta.StoredMetadata}) diff --git a/lib/instances/query_test.go b/lib/instances/query_test.go index ab3db29c3..b3dbfba41 100644 --- a/lib/instances/query_test.go +++ b/lib/instances/query_test.go @@ -42,6 +42,47 @@ func TestListInstancesForReconcileFailsOnInvalidMetadata(t *testing.T) { assert.Equal(t, "valid", listed[0].Id) } +// A concurrent delete can remove an instance between the reconcile listing +// and its metadata load. A vanished record cannot claim a VF, so it must be +// skipped like the release claim scan does — failing instead would zero the +// grace-period retry and silently disable the vendor VFIO sweep whenever it +// races a delete. +func TestListInstancesForReconcileSkipsInstanceDeletedDuringListing(t *testing.T) { + m := &manager{paths: paths.New(t.TempDir())} + + for _, id := range []string{"aaa-ghost", "zzz-live"} { + require.NoError(t, m.ensureDirectories(id)) + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: id, + Name: id, + CreatedAt: time.Now(), + DataDir: m.paths.InstanceDir(id), + }})) + } + + // loadMetadata takes the snapshot-alias read lock, so holding the + // mutation lock parks the reconcile between listing and loading — the + // window a concurrent delete lands in. + unlock := hypervisor.LockSnapshotSourceAliasMutation() + type result struct { + listed []Instance + err error + } + done := make(chan result, 1) + go func() { + listed, err := m.ListInstancesForReconcile(context.Background()) + done <- result{listed, err} + }() + time.Sleep(100 * time.Millisecond) + require.NoError(t, os.Remove(m.paths.InstanceMetadata("aaa-ghost"))) + unlock() + + res := <-done + require.NoError(t, res.err) + require.Len(t, res.listed, 1) + assert.Equal(t, "zzz-live", res.listed[0].Id) +} + func TestParseExitSentinelLine(t *testing.T) { t.Parallel() tests := []struct { From f255bb2b78a27d5e11647f9739ae0db631b273e2 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Fri, 21 Aug 2026 16:31:49 +0000 Subject: [PATCH 51/76] Count abandoned orphaned vGPU releases Giving up on an orphaned release leaves the VF allocated while /resources still advertises it, until startup reconciliation or manual remediation. That was visible only as a log line; count it so capacity leaks can alert. --- lib/instances/metrics.go | 21 +++++++++++++++++++++ lib/instances/vgpu_orphan.go | 1 + 2 files changed, 22 insertions(+) diff --git a/lib/instances/metrics.go b/lib/instances/metrics.go index 1ada5ac1e..bdffbdbce 100644 --- a/lib/instances/metrics.go +++ b/lib/instances/metrics.go @@ -94,6 +94,7 @@ type Metrics struct { lifecycleEventsDroppedTotal metric.Int64Counter forkMemFileShareFallbacksTotal metric.Int64Counter ttlReaperDeletionsTotal metric.Int64Counter + vgpuOrphanReleasesAbandonedTotal metric.Int64Counter tracer trace.Tracer } @@ -270,6 +271,14 @@ func newInstanceMetrics(meter metric.Meter, tracer trace.Tracer, m *manager) (*M return nil, err } + vgpuOrphanReleasesAbandonedTotal, err := meter.Int64Counter( + "hypeman_instances_vgpu_orphan_releases_abandoned_total", + metric.WithDescription("Total orphaned vGPU release retries that gave up, leaving the VF allocated until startup reconciliation or manual remediation"), + ) + if err != nil { + return nil, err + } + // Register observable gauge for instance counts by state instancesTotal, err := meter.Int64ObservableGauge( "hypeman_instances_total", @@ -464,6 +473,7 @@ func newInstanceMetrics(meter metric.Meter, tracer trace.Tracer, m *manager) (*M lifecycleEventsDroppedTotal: lifecycleEventsDroppedTotal, forkMemFileShareFallbacksTotal: forkMemFileShareFallbacksTotal, ttlReaperDeletionsTotal: ttlReaperDeletionsTotal, + vgpuOrphanReleasesAbandonedTotal: vgpuOrphanReleasesAbandonedTotal, tracer: tracer, }, nil } @@ -563,6 +573,17 @@ func (m *manager) recordTimeToRunning(ctx context.Context, stored *StoredMetadat m.metrics.timeToRunning.Record(ctx, duration, metric.WithAttributes(attrs...)) } +// recordVGPUOrphanReleaseAbandoned records an orphaned vGPU release retry +// loop giving up: the VF stays allocated (capacity silently reduced) until +// startup reconciliation or manual remediation, so it must be visible beyond +// a log line. +func (m *manager) recordVGPUOrphanReleaseAbandoned(ctx context.Context) { + if m.metrics == nil { + return + } + m.metrics.vgpuOrphanReleasesAbandonedTotal.Add(ctx, 1) +} + // recordStateTransition records a state transition with hypervisor label. func (m *manager) recordStateTransition(ctx context.Context, fromState, toState string, hvType hypervisor.Type) { if m.metrics == nil { diff --git a/lib/instances/vgpu_orphan.go b/lib/instances/vgpu_orphan.go index 5f3fc2b38..b3ff2b33b 100644 --- a/lib/instances/vgpu_orphan.go +++ b/lib/instances/vgpu_orphan.go @@ -67,6 +67,7 @@ func (m *manager) retryOrphanedVGPURelease(ctx context.Context, stored StoredMet "instance_id", stored.Id, "device_path", path, "attempt", attempt) return } + m.recordVGPUOrphanReleaseAbandoned(ctx) log.ErrorContext(ctx, "giving up on orphaned vGPU release; VF stays allocated until startup reconciliation or manual remediation", "instance_id", stored.Id, "device_path", path, "attempts", orphanedVGPUReleaseMaxAttempts) } From 3644fbe1d799b453ad3dc08ddf3a9ca347edfd62 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Fri, 21 Aug 2026 17:52:25 +0000 Subject: [PATCH 52/76] Define the vGPU retention-stub rejection once Start, fork, and snapshot each carried a verbatim copy of the rejection error and its rationale; the stub fill in create's cleanup closure duplicated retainedVGPUFromCreateError field-for-field. One error value and one device-to-stub helper replace the copies. --- lib/instances/create.go | 8 +------- lib/instances/fork.go | 5 +---- lib/instances/snapshot.go | 6 +----- lib/instances/start.go | 5 +---- lib/instances/vgpu.go | 16 ++++++++++++---- 5 files changed, 16 insertions(+), 24 deletions(-) diff --git a/lib/instances/create.go b/lib/instances/create.go index eefd0a7ce..66c988b4f 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -354,13 +354,7 @@ func (m *manager) createInstance( log.WarnContext(ctx, "failed to destroy vGPU on cleanup", "instance_id", id, "uuid", gpuDevice.MdevUUID, "error", err) retainedVGPU = stored if retainedVGPU == nil { - stub := retentionStub() - stub.GPUProfile = gpuDevice.ProfileName - stub.GPUFramework = gpuDevice.Framework - stub.GPUDevicePath = gpuDevice.SysfsPath - stub.GPUMdevUUID = gpuDevice.MdevUUID - stub.GPUAssignedAt = gpuAssignedAt - retainedVGPU = &stub + retainedVGPU = retainedVGPUFromDevice(retentionStub(), gpuDevice, *gpuAssignedAt) } } }) diff --git a/lib/instances/fork.go b/lib/instances/fork.go index 266299e96..08ce4014a 100644 --- a/lib/instances/fork.go +++ b/lib/instances/fork.go @@ -220,10 +220,7 @@ func (m *manager) forkInstanceFromStoppedOrStandby(ctx context.Context, id strin return nil, false, fmt.Errorf("%w: cannot fork from state %s (must be Stopped or Standby)", ErrInvalidState, source.State) } if stored.GPURetainedForCleanup { - // A delete-only retention stub from a failed create has no boot - // configuration, so a fork of it could never boot. Delete the stub to - // release its retained vGPU assignment. - return nil, false, fmt.Errorf("%w: instance retains a vGPU assignment from a failed create and has no boot configuration; delete it to release the assignment", ErrInvalidState) + return nil, false, errVGPURetentionStub } if !supportValidated { diff --git a/lib/instances/snapshot.go b/lib/instances/snapshot.go index 297c7ef03..e30192ee9 100644 --- a/lib/instances/snapshot.go +++ b/lib/instances/snapshot.go @@ -67,11 +67,7 @@ func (m *manager) createSnapshot(ctx context.Context, id string, req CreateSnaps stored := &meta.StoredMetadata if stored.GPURetainedForCleanup { - // A delete-only retention stub from a failed create has no boot - // configuration, so a snapshot of it could never be restored or - // forked into a bootable instance. Delete the stub to release its - // retained vGPU assignment. - return nil, fmt.Errorf("%w: instance retains a vGPU assignment from a failed create and has no boot configuration; delete it to release the assignment", ErrInvalidState) + return nil, errVGPURetentionStub } if err := validateForkVolumeSafety(stored.Volumes); err != nil { return nil, fmt.Errorf("%w: snapshot requires readonly volume attachments: %v", ErrNotSupported, err) diff --git a/lib/instances/start.go b/lib/instances/start.go index 44dc50216..c7eb85493 100644 --- a/lib/instances/start.go +++ b/lib/instances/start.go @@ -48,11 +48,8 @@ func (m *manager) startInstance( return nil, fmt.Errorf("%w: cannot start from state %s, must be Stopped", ErrInvalidState, inst.State) } if stored.GPURetainedForCleanup { - // A delete-only retention stub from a failed create: it carries no - // boot configuration, so starting it would release the retained VF - // and then boot an incomplete record. Delete retries the release. log.ErrorContext(ctx, "refusing to start vGPU retention record", "instance_id", id) - return nil, fmt.Errorf("%w: instance retains a vGPU assignment from a failed create and has no boot configuration; delete it to release the assignment", ErrInvalidState) + return nil, errVGPURetentionStub } // Release any assignment retained by an earlier failed release and diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index b38fb5225..dcb6cff4d 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -33,6 +33,11 @@ func (e *VGPUCleanupPendingError) Error() string { func (e *VGPUCleanupPendingError) Unwrap() error { return e.Err } +// errVGPURetentionStub rejects every lifecycle verb except delete on a +// retention stub from a failed create: the record has no boot configuration, +// and only delete retries the release of its retained assignment. +var errVGPURetentionStub = fmt.Errorf("%w: instance retains a vGPU assignment from a failed create and has no boot configuration; delete it to release the assignment", ErrInvalidState) + func (m *manager) createVGPUDevice(ctx context.Context, profileName, instanceID string) (*devices.VGPUDevice, error) { create := m.createVGPU if create == nil { @@ -58,11 +63,14 @@ func retainedVGPUFromCreateError(stub StoredMetadata, assignedAt time.Time, err if !ok { return nil } + return retainedVGPUFromDevice(stub, device, assignedAt) +} + +// retainedVGPUFromDevice fills stub with device's assignment fields so a +// failed rollback release retains a recognizable, deletable record. +func retainedVGPUFromDevice(stub StoredMetadata, device *devices.VGPUDevice, assignedAt time.Time) *StoredMetadata { stub.GPUProfile = device.ProfileName - stub.GPUFramework = device.Framework - stub.GPUDevicePath = device.SysfsPath - stub.GPUMdevUUID = device.MdevUUID - stub.GPUAssignedAt = &assignedAt + setStoredVGPUDevice(&stub, device, assignedAt) return &stub } From b4446633e46f2ef5bd1947c186bb8bbfd68340b9 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Fri, 21 Aug 2026 17:52:25 +0000 Subject: [PATCH 53/76] Render pending vGPU cleanup responses through one helper The create and start handlers carried near-identical 20-line blocks deriving the vgpu_cleanup_pending message and inner error detail, differing only in the verb and release guidance. --- cmd/api/api/instances.go | 48 ++++++++++++++++++++-------------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/cmd/api/api/instances.go b/cmd/api/api/instances.go index a277b5476..1c502c928 100644 --- a/cmd/api/api/instances.go +++ b/cmd/api/api/instances.go @@ -368,19 +368,11 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst // errors.Is case would match the cause and hide the pending vGPU cleanup. case errors.As(err, &vgpuPending): log.ErrorContext(ctx, "failed to create instance", "error", err, "image", request.Body.Image) - message := fmt.Sprintf("failed to create instance: %v; vGPU release failed during rollback and instance %s retains the assignment, delete it to retry", vgpuPending.Err, vgpuPending.InstanceID) - innerCode := "vgpu_retained_instance" - if !vgpuPending.Retained { - message = fmt.Sprintf("failed to create instance: %v; vGPU release failed during rollback and the retention record for instance %s could not be saved; the assignment is recovered on the next startup reconcile", vgpuPending.Err, vgpuPending.InstanceID) - innerCode = "vgpu_unretained_instance" - } + message, inner := vgpuCleanupPendingDetail(vgpuPending, "create", "delete it to retry") return oapi.CreateInstance500JSONResponse{ - Code: "vgpu_cleanup_pending", - Message: message, - InnerError: &oapi.ErrorDetail{ - Code: lo.ToPtr(innerCode), - Message: lo.ToPtr(vgpuPending.InstanceID), - }, + Code: "vgpu_cleanup_pending", + Message: message, + InnerError: inner, }, nil case errors.Is(err, instances.ErrImageNotReady): return oapi.CreateInstance400JSONResponse{ @@ -443,6 +435,22 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst return oapi.CreateInstance201JSONResponse(instanceToOAPI(*inst)), nil } +// vgpuCleanupPendingDetail renders a pending vGPU cleanup into the message +// and inner error detail shared by the create and start handlers. The +// retained guidance names the verb-specific way to release the assignment. +func vgpuCleanupPendingDetail(pending *instances.VGPUCleanupPendingError, action, retainedGuidance string) (string, *oapi.ErrorDetail) { + message := fmt.Sprintf("failed to %s instance: %v; vGPU release failed during rollback and instance %s retains the assignment, %s", action, pending.Err, pending.InstanceID, retainedGuidance) + innerCode := "vgpu_retained_instance" + if !pending.Retained { + message = fmt.Sprintf("failed to %s instance: %v; vGPU release failed during rollback and the retention record for instance %s could not be saved; the assignment is recovered on the next startup reconcile", action, pending.Err, pending.InstanceID) + innerCode = "vgpu_unretained_instance" + } + return message, &oapi.ErrorDetail{ + Code: lo.ToPtr(innerCode), + Message: lo.ToPtr(pending.InstanceID), + } +} + // GetInstance gets instance details // The id parameter can be an instance ID, name, or ID prefix // Note: Resolution is handled by ResolveResource middleware @@ -859,19 +867,11 @@ func (s *ApiService) StartInstance(ctx context.Context, request oapi.StartInstan // errors.Is case would match the cause and hide the pending vGPU cleanup. case errors.As(err, &vgpuPending): log.ErrorContext(ctx, "failed to start instance", "error", err) - message := fmt.Sprintf("failed to start instance: %v; vGPU release failed during rollback and instance %s retains the assignment, delete it or retry start to release it", vgpuPending.Err, vgpuPending.InstanceID) - innerCode := "vgpu_retained_instance" - if !vgpuPending.Retained { - message = fmt.Sprintf("failed to start instance: %v; vGPU release failed during rollback and the retention record for instance %s could not be saved; the assignment is recovered on the next startup reconcile", vgpuPending.Err, vgpuPending.InstanceID) - innerCode = "vgpu_unretained_instance" - } + message, inner := vgpuCleanupPendingDetail(vgpuPending, "start", "delete it or retry start to release it") return oapi.StartInstance500JSONResponse{ - Code: "vgpu_cleanup_pending", - Message: message, - InnerError: &oapi.ErrorDetail{ - Code: lo.ToPtr(innerCode), - Message: lo.ToPtr(vgpuPending.InstanceID), - }, + Code: "vgpu_cleanup_pending", + Message: message, + InnerError: inner, }, nil case errors.Is(err, instances.ErrInvalidState): return oapi.StartInstance409JSONResponse{ From 4d21b0ba5c38b7e44996efc909d023afcf26b65e Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Mon, 24 Aug 2026 01:27:54 +0000 Subject: [PATCH 54/76] Narrow the vGPU reconcile interface --- cmd/api/main.go | 10 +++++++++- cmd/api/main_test.go | 5 +++++ lib/instances/manager.go | 1 - 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/cmd/api/main.go b/cmd/api/main.go index 71e147ecf..a4db0f3e9 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -185,8 +185,16 @@ func configureUFFDGraduationController(cfg *config.Config, instanceManager insta }, logger), nil } +type vgpuReconcileInstanceLister interface { + ListInstancesForReconcile(context.Context) ([]instances.Instance, error) +} + func liveInstanceVGPUDevicePaths(ctx context.Context, instanceManager instances.Manager) (map[string]struct{}, time.Duration, error) { - allInstances, err := instanceManager.ListInstancesForReconcile(ctx) + lister, ok := instanceManager.(vgpuReconcileInstanceLister) + if !ok { + return nil, 0, errors.New("instance manager does not support vGPU reconcile inventory") + } + allInstances, err := lister.ListInstancesForReconcile(ctx) if err != nil { return nil, 0, err } diff --git a/cmd/api/main_test.go b/cmd/api/main_test.go index 85c132814..0718b4cf3 100644 --- a/cmd/api/main_test.go +++ b/cmd/api/main_test.go @@ -351,6 +351,11 @@ func (s vgpuReconcileManagerStub) ListInstancesForReconcile(context.Context) ([] return s.list, nil } +func TestLiveInstanceVGPUDevicePathsRequiresReconcileInventory(t *testing.T) { + _, _, err := liveInstanceVGPUDevicePaths(context.Background(), struct{ instances.Manager }{}) + require.ErrorContains(t, err, "does not support vGPU reconcile inventory") +} + func TestLiveInstanceVGPUDevicePathsBoundsStartupProtection(t *testing.T) { dead := exec.Command("true") require.NoError(t, dead.Run()) diff --git a/lib/instances/manager.go b/lib/instances/manager.go index 54e9b00d6..f3da80470 100644 --- a/lib/instances/manager.go +++ b/lib/instances/manager.go @@ -29,7 +29,6 @@ import ( type Manager interface { ListInstances(ctx context.Context, filter *ListInstancesFilter) ([]Instance, error) - ListInstancesForReconcile(ctx context.Context) ([]Instance, error) ListSnapshots(ctx context.Context, filter *ListSnapshotsFilter) ([]Snapshot, error) GetSnapshot(ctx context.Context, snapshotID string) (*Snapshot, error) CreateInstance(ctx context.Context, req CreateInstanceRequest) (*Instance, error) From 34deb09f5fe4957a5e8e6ce24f23f21e7e42304f Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Mon, 24 Aug 2026 14:54:32 +0000 Subject: [PATCH 55/76] Encapsulate vGPU rollback retention --- lib/instances/create.go | 73 ++----------------- lib/instances/lifecycle_noop_test.go | 2 +- lib/instances/start.go | 17 ++--- lib/instances/vgpu.go | 95 +++++++++++++----------- lib/instances/vgpu_retention.go | 104 +++++++++++++++++++++++++++ lib/instances/vgpu_test.go | 46 ++++++++++-- 6 files changed, 212 insertions(+), 125 deletions(-) create mode 100644 lib/instances/vgpu_retention.go diff --git a/lib/instances/create.go b/lib/instances/create.go index 66c988b4f..f4ece0100 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -279,21 +279,13 @@ func (m *manager) createInstance( var gpuMdevUUID string var gpuAssignedAt *time.Time var stored *StoredMetadata - var retainedVGPU *StoredMetadata + retention := vgpuRetention{instanceID: id} // Setup cleanup stack early so device attachment errors trigger cleanup. - // When rollback cannot release a vGPU assignment, report whether its - // retention record was persisted. The wrapping defer is registered first - // so it runs after cu.Clean has attempted to retain the metadata. - vgpuPersisted := false - defer func() { - if retErr != nil && retainedVGPU != nil { - retErr = &VGPUCleanupPendingError{InstanceID: id, Retained: vgpuPersisted, Err: retErr} - } - }() + defer retention.deferWrapPending(&retErr) cu := cleanup.Make(func() { log.DebugContext(ctx, "cleaning up instance on error", "instance_id", id) - vgpuPersisted = m.cleanupFailedCreate(ctx, id, retainedVGPU) + m.persistVGPURetention(ctx, &retention) }) defer cu.Clean() @@ -329,7 +321,7 @@ func (m *manager) createInstance( log.InfoContext(ctx, "creating vGPU", "instance_id", id, "profile", req.GPU.Profile) gpuDevice, err = m.createVGPUDevice(ctx, req.GPU.Profile, id) if err != nil { - retainedVGPU = retainedVGPUFromCreateError(retentionStub(), m.nowUTC(), err) + retention.retainFromCreateError(retentionStub(), m.nowUTC(), err) log.ErrorContext(ctx, "failed to create vGPU", "profile", req.GPU.Profile, "error", err) return nil, wrapCreateVGPUErr(req.GPU.Profile, err) } @@ -352,9 +344,9 @@ func (m *manager) createInstance( } if err := m.destroyVGPUAssignment(ctx, assignment); err != nil { log.WarnContext(ctx, "failed to destroy vGPU on cleanup", "instance_id", id, "uuid", gpuDevice.MdevUUID, "error", err) - retainedVGPU = stored - if retainedVGPU == nil { - retainedVGPU = retainedVGPUFromDevice(retentionStub(), gpuDevice, *gpuAssignedAt) + retention.retain(stored) + if stored == nil { + retention.retainFromDevice(retentionStub(), gpuDevice, *gpuAssignedAt) } } }) @@ -647,57 +639,6 @@ func resolveCreateExpiration(req CreateInstanceRequest, now time.Time) (*time.Ti return &expiresAt, nil } -// cleanupFailedCreate reports whether the retention record for a vGPU -// assignment whose release failed during rollback was persisted. -func (m *manager) cleanupFailedCreate(ctx context.Context, id string, retainedVGPU *StoredMetadata) bool { - if retainedVGPU == nil { - m.deleteInstanceData(id) - return false - } - - log := logger.FromContext(ctx) - retentionSurvives := func() bool { - meta, err := m.loadMetadata(id) - if err == nil && storedVGPUDevicePath(&meta.StoredMetadata) != "" { - return true - } - if err := m.deleteInstanceData(id); err != nil { - log.ErrorContext(ctx, "failed to delete stale instance data after retention failure", "instance_id", id, "error", err) - } - return false - } - if err := m.ensureDirectories(id); err != nil { - log.ErrorContext(ctx, "failed to retain instance data after vGPU cleanup failure", "instance_id", id, "error", err) - return retentionSurvives() - } - // Retain identity fields so the instance lists as a recognizable, - // deletable record rather than a nameless phantom, but drop resource - // claims (network, volumes, devices) that rollback already released. - retained := StoredMetadata{ - Id: id, - Name: retainedVGPU.Name, - Image: retainedVGPU.Image, - ResolvedImage: retainedVGPU.ResolvedImage, - Platform: retainedVGPU.Platform, - CreatedAt: retainedVGPU.CreatedAt, - HypervisorType: retainedVGPU.HypervisorType, - HypervisorVersion: retainedVGPU.HypervisorVersion, - SocketPath: retainedVGPU.SocketPath, - DataDir: retainedVGPU.DataDir, - GPUProfile: retainedVGPU.GPUProfile, - GPUFramework: retainedVGPU.GPUFramework, - GPUDevicePath: retainedVGPU.GPUDevicePath, - GPUMdevUUID: retainedVGPU.GPUMdevUUID, - GPUAssignedAt: retainedVGPU.GPUAssignedAt, - GPURetainedForCleanup: true, - } - if err := m.saveMetadata(&metadata{StoredMetadata: retained}); err != nil { - log.ErrorContext(ctx, "failed to retain vGPU assignment metadata after cleanup failure", "instance_id", id, "error", err) - return retentionSurvives() - } - return true -} - // validateCreateRequest validates the create instance request. // The request is mutated in-place to persist normalized egress/credential policy fields. func validateCreateRequest(req *CreateInstanceRequest) error { diff --git a/lib/instances/lifecycle_noop_test.go b/lib/instances/lifecycle_noop_test.go index 46f891379..a43efabb0 100644 --- a/lib/instances/lifecycle_noop_test.go +++ b/lib/instances/lifecycle_noop_test.go @@ -201,7 +201,7 @@ func TestDeletePersistsVGPUReleaseBeforeTeardown(t *testing.T) { // A failed create whose vGPU release also failed retains a minimal // GPU-fields-only stub, and the API tells the caller to delete it to retry // the release. Exercise that recovery path against the exact stub shape -// cleanupFailedCreate writes. +// persistVGPURetention writes. func TestDeleteReleasesRetainedCreateStub(t *testing.T) { p := paths.New(t.TempDir()) var destroyed []devices.VGPUAssignment diff --git a/lib/instances/start.go b/lib/instances/start.go index c7eb85493..a79663953 100644 --- a/lib/instances/start.go +++ b/lib/instances/start.go @@ -121,16 +121,8 @@ func (m *manager) startInstance( } // Setup cleanup stack for automatic rollback on errors - // Registered before cu.Clean so it runs after cleanup and can report a - // vGPU assignment that rollback failed to destroy, matching create's - // vgpu_cleanup_pending contract. - vgpuRetained := false - vgpuRetentionPersisted := false - defer func() { - if retErr != nil && vgpuRetained { - retErr = &VGPUCleanupPendingError{InstanceID: id, Retained: vgpuRetentionPersisted, Err: retErr} - } - }() + retention := vgpuRetention{instanceID: id} + defer retention.deferWrapPending(&retErr) cu := cleanup.Make(func() {}) defer cu.Clean() @@ -204,7 +196,10 @@ func (m *manager) startInstance( log.InfoContext(ctx, "created vGPU", "instance_id", id, "profile", stored.GPUProfile, "uuid", device.MdevUUID) // Add vGPU cleanup to stack cu.Add(func() { - vgpuRetained, vgpuRetentionPersisted = m.cleanupStartVGPU(ctx, id, device, assignedAt, rollbackMeta) + retained, persisted := m.cleanupStartVGPU(ctx, id, device, assignedAt, rollbackMeta) + if retained { + retention.markRetained(persisted) + } }) if err := m.saveMetadata(meta); err != nil { log.ErrorContext(ctx, "failed to save metadata after vGPU creation", "instance_id", id, "error", err) diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index dcb6cff4d..3561cbc5a 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -54,24 +54,18 @@ func vgpuDevicePendingCleanup(err error) (*devices.VGPUDevice, bool) { return &pending.Device, true } -// retainedVGPUFromCreateError fills stub with the pending device's assignment -// fields when err carries a failed device-layer cleanup. The caller provides -// identity fields on stub so the retained record lists as a recognizable, -// deletable instance. -func retainedVGPUFromCreateError(stub StoredMetadata, assignedAt time.Time, err error) *StoredMetadata { - device, ok := vgpuDevicePendingCleanup(err) - if !ok { - return nil +func vgpuAssignmentLiveness(stored *StoredMetadata, now time.Time, livePID bool) (live bool, graceRemaining time.Duration) { + if stored.HypervisorPID != nil && livePID { + return true, 0 } - return retainedVGPUFromDevice(stub, device, assignedAt) -} - -// retainedVGPUFromDevice fills stub with device's assignment fields so a -// failed rollback release retains a recognizable, deletable record. -func retainedVGPUFromDevice(stub StoredMetadata, device *devices.VGPUDevice, assignedAt time.Time) *StoredMetadata { - stub.GPUProfile = device.ProfileName - setStoredVGPUDevice(&stub, device, assignedAt) - return &stub + if stored.GPUAssignedAt == nil { + return false, 0 + } + remaining := VGPUAssignmentStartupGracePeriod - now.Sub(*stored.GPUAssignedAt) + if remaining <= 0 { + return false, 0 + } + return true, remaining } func (m *manager) destroyVGPUAssignment(ctx context.Context, assignment devices.VGPUAssignment) error { @@ -96,15 +90,9 @@ func clearStoredVGPUDevice(stored *StoredMetadata) { stored.GPUAssignedAt = nil } -// cleanupStartVGPU wholesale-restores the pre-start metadata snapshot. The -// cleanup stack is LIFO, so cleanups registered after this one run before it -// and this restore would clobber anything they persisted; it is safe only -// while no such cleanup writes metadata and the instance lock serializes -// start. Violating that requires switching to targeted field restores. -// -// It reports whether the assignment was retained after a failed destroy and -// whether that retention record was persisted, so start can surface the -// pending cleanup as a typed error like create does. +// cleanupStartVGPU reports whether the assignment was retained after a failed +// destroy and whether that retention record was persisted, so start can surface +// the pending cleanup as a typed error like create does. func (m *manager) cleanupStartVGPU(ctx context.Context, instanceID string, device *devices.VGPUDevice, assignedAt time.Time, rollbackMeta metadata) (retained, persisted bool) { logger.FromContext(ctx).DebugContext(ctx, "destroying vGPU on cleanup", "instance_id", instanceID, "uuid", device.MdevUUID) assignment := devices.VGPUAssignment{ @@ -113,14 +101,20 @@ func (m *manager) cleanupStartVGPU(ctx context.Context, instanceID string, devic MdevUUID: device.MdevUUID, InstanceID: instanceID, } - cleanupMeta := rollbackMeta + cleanupMeta, err := m.loadMetadata(instanceID) + if err != nil { + logger.FromContext(ctx).WarnContext(ctx, "failed to load current metadata for vGPU cleanup; restoring rollback snapshot", "instance_id", instanceID, "error", err) + cleanupMeta = &rollbackMeta + } else { + restoreStartMutatedFields(&cleanupMeta.StoredMetadata, &rollbackMeta.StoredMetadata) + } releaseErr := m.destroyVGPUAssignment(ctx, assignment) if releaseErr != nil { logger.FromContext(ctx).WarnContext(ctx, "failed to destroy vGPU on cleanup", "instance_id", instanceID, "uuid", device.MdevUUID, "error", releaseErr) setStoredVGPUDevice(&cleanupMeta.StoredMetadata, device, assignedAt) retained = true } - if err := m.saveMetadata(&cleanupMeta); err != nil { + if err := m.saveMetadata(cleanupMeta); err != nil { message := "failed to save metadata after vGPU cleanup" if releaseErr != nil { message = "failed to retain vGPU assignment metadata after cleanup failure" @@ -141,6 +135,27 @@ func (m *manager) cleanupStartVGPU(ctx context.Context, instanceID string, devic return retained, retained } +// restoreStartMutatedFields must cover every field start mutates before the +// vGPU cleanup runs. +func restoreStartMutatedFields(dst, src *StoredMetadata) { + dst.HypervisorPID = src.HypervisorPID + dst.HypervisorStartTime = src.HypervisorStartTime + dst.HypervisorBootID = src.HypervisorBootID + dst.ExitCode = src.ExitCode + dst.ExitMessage = src.ExitMessage + dst.ProgramStartedAt = src.ProgramStartedAt + dst.GuestAgentReadyAt = src.GuestAgentReadyAt + dst.Entrypoint = src.Entrypoint + dst.Cmd = src.Cmd + dst.IP = src.IP + dst.MAC = src.MAC + dst.GPUFramework = src.GPUFramework + dst.GPUDevicePath = src.GPUDevicePath + dst.GPUMdevUUID = src.GPUMdevUUID + dst.GPUAssignedAt = src.GPUAssignedAt + dst.StartedAt = src.StartedAt +} + func (m *manager) releaseStoredVGPU(ctx context.Context, stored *StoredMetadata) error { path := storedVGPUDevicePath(stored) if path != "" { @@ -206,23 +221,21 @@ func (m *manager) vgpuAssignmentClaimedByLiveInstance(ctx context.Context, exclu if storedVGPUDevicePath(stored) != devicePath { continue } - if stored.HypervisorPID == nil { - if stored.GPUAssignedAt == nil || time.Since(*stored.GPUAssignedAt) >= VGPUAssignmentStartupGracePeriod { - continue + pid := 0 + if stored.HypervisorPID != nil { + pid, err = resolveLiveHypervisorPID(stored.HypervisorProcessIdentity, stored.SocketPath) + if err != nil { + return false, fmt.Errorf("cannot confirm liveness of vGPU claimant %s on %s: %w", id, devicePath, err) } - return false, fmt.Errorf("cannot confirm liveness of recent vGPU claimant %s on %s: no persisted hypervisor PID", id, devicePath) } - pid, err := resolveLiveHypervisorPID(stored.HypervisorProcessIdentity, stored.SocketPath) - if err != nil { - return false, fmt.Errorf("cannot confirm liveness of vGPU claimant %s on %s: %w", id, devicePath, err) - } - if pid > 0 { + live, remaining := vgpuAssignmentLiveness(stored, time.Now(), pid > 0) + if pid > 0 && live { return true, nil } - // A dead PID with a recent assignment gets the same bounded grace as - // startup reconcile protection, so the two guards agree in the - // fail-closed direction while a mid-boot claimant hydrates. - if stored.GPUAssignedAt != nil && time.Since(*stored.GPUAssignedAt) < VGPUAssignmentStartupGracePeriod { + if remaining > 0 { + if stored.HypervisorPID == nil { + return false, fmt.Errorf("cannot confirm liveness of recent vGPU claimant %s on %s: no persisted hypervisor PID", id, devicePath) + } return false, fmt.Errorf("cannot confirm liveness of recent vGPU claimant %s on %s: recorded hypervisor is not running", id, devicePath) } } diff --git a/lib/instances/vgpu_retention.go b/lib/instances/vgpu_retention.go new file mode 100644 index 000000000..a1a1aae0d --- /dev/null +++ b/lib/instances/vgpu_retention.go @@ -0,0 +1,104 @@ +package instances + +import ( + "context" + "time" + + "github.com/kernel/hypeman/lib/devices" + "github.com/kernel/hypeman/lib/logger" +) + +type vgpuRetention struct { + instanceID string + stub *StoredMetadata + retained bool + persisted bool +} + +func (r *vgpuRetention) retainFromCreateError(stub StoredMetadata, assignedAt time.Time, err error) { + device, ok := vgpuDevicePendingCleanup(err) + if !ok { + return + } + r.retainFromDevice(stub, device, assignedAt) +} + +func (r *vgpuRetention) retainFromDevice(stub StoredMetadata, device *devices.VGPUDevice, assignedAt time.Time) { + stub.GPUProfile = device.ProfileName + setStoredVGPUDevice(&stub, device, assignedAt) + r.stub = &stub + r.retained = true +} + +func (r *vgpuRetention) retain(stub *StoredMetadata) { + r.stub = stub + r.retained = stub != nil +} + +func (r *vgpuRetention) markRetained(persisted bool) { + r.retained = true + r.persisted = persisted +} + +func (r *vgpuRetention) wrapPending(err error) error { + if err == nil || !r.retained { + return err + } + return &VGPUCleanupPendingError{InstanceID: r.instanceID, Retained: r.persisted, Err: err} +} + +// deferWrapPending must be deferred before cleanup so it observes retention +// state recorded by rollback. +func (r *vgpuRetention) deferWrapPending(retErr *error) { + *retErr = r.wrapPending(*retErr) +} + +func (m *manager) persistVGPURetention(ctx context.Context, retention *vgpuRetention) { + if retention.stub == nil { + m.deleteInstanceData(retention.instanceID) + return + } + + id := retention.instanceID + retainedVGPU := retention.stub + log := logger.FromContext(ctx) + retentionSurvives := func() bool { + meta, err := m.loadMetadata(id) + if err == nil && storedVGPUDevicePath(&meta.StoredMetadata) != "" { + return true + } + if err := m.deleteInstanceData(id); err != nil { + log.ErrorContext(ctx, "failed to delete stale instance data after retention failure", "instance_id", id, "error", err) + } + return false + } + if err := m.ensureDirectories(id); err != nil { + log.ErrorContext(ctx, "failed to retain instance data after vGPU cleanup failure", "instance_id", id, "error", err) + retention.persisted = retentionSurvives() + return + } + retained := StoredMetadata{ + Id: id, + Name: retainedVGPU.Name, + Image: retainedVGPU.Image, + ResolvedImage: retainedVGPU.ResolvedImage, + Platform: retainedVGPU.Platform, + CreatedAt: retainedVGPU.CreatedAt, + HypervisorType: retainedVGPU.HypervisorType, + HypervisorVersion: retainedVGPU.HypervisorVersion, + SocketPath: retainedVGPU.SocketPath, + DataDir: retainedVGPU.DataDir, + GPUProfile: retainedVGPU.GPUProfile, + GPUFramework: retainedVGPU.GPUFramework, + GPUDevicePath: retainedVGPU.GPUDevicePath, + GPUMdevUUID: retainedVGPU.GPUMdevUUID, + GPUAssignedAt: retainedVGPU.GPUAssignedAt, + GPURetainedForCleanup: true, + } + if err := m.saveMetadata(&metadata{StoredMetadata: retained}); err != nil { + log.ErrorContext(ctx, "failed to retain vGPU assignment metadata after cleanup failure", "instance_id", id, "error", err) + retention.persisted = retentionSurvives() + return + } + retention.persisted = true +} diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index 1d0e4e3bf..c5ef4911b 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -18,6 +18,19 @@ import ( "github.com/stretchr/testify/require" ) +func persistTestVGPURetention(m *manager, ctx context.Context, id string, stub *StoredMetadata) bool { + retention := vgpuRetention{instanceID: id} + retention.retain(stub) + m.persistVGPURetention(ctx, &retention) + return retention.persisted +} + +func retainedVGPUFromCreateErrorForTest(stub StoredMetadata, assignedAt time.Time, err error) *StoredMetadata { + retention := vgpuRetention{} + retention.retainFromCreateError(stub, assignedAt, err) + return retention.stub +} + func TestCleanupFailedCreateRetainsVGPUAssignment(t *testing.T) { t.Parallel() @@ -38,7 +51,7 @@ func TestCleanupFailedCreateRetainsVGPUAssignment(t *testing.T) { DataDir: m.paths.InstanceDir("failed-create"), } - assert.True(t, m.cleanupFailedCreate(context.Background(), stored.Id, stored)) + assert.True(t, persistTestVGPURetention(m, context.Background(), stored.Id, stored)) retained, err := m.loadMetadata(stored.Id) require.NoError(t, err) @@ -67,7 +80,7 @@ func TestCleanupFailedCreateDeletesDataWithoutRetainedVGPU(t *testing.T) { m := &manager{paths: paths.New(t.TempDir())} require.NoError(t, m.ensureDirectories("failed-create")) - assert.False(t, m.cleanupFailedCreate(context.Background(), "failed-create", nil)) + assert.False(t, persistTestVGPURetention(m, context.Background(), "failed-create", nil)) _, err := m.loadMetadata("failed-create") require.Error(t, err) } @@ -85,7 +98,7 @@ func TestCleanupFailedCreateReportsUnpersistedRetention(t *testing.T) { GPUFramework: devices.VGPUFrameworkVendorVFIO, GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", } - assert.False(t, m.cleanupFailedCreate(context.Background(), stored.Id, stored)) + assert.False(t, persistTestVGPURetention(m, context.Background(), stored.Id, stored)) _, err := m.loadMetadata(id) require.Error(t, err) } @@ -109,13 +122,31 @@ func TestCleanupFailedCreateReportsRetainedWhenFullMetadataSurvives(t *testing.T require.NoError(t, os.Chmod(instanceDir, 0o555)) t.Cleanup(func() { _ = os.Chmod(instanceDir, 0o755) }) - assert.True(t, m.cleanupFailedCreate(context.Background(), id, stored)) + assert.True(t, persistTestVGPURetention(m, context.Background(), id, stored)) retained, err := m.loadMetadata(id) require.NoError(t, err) assert.Equal(t, stored.GPUFramework, retained.GPUFramework) assert.Equal(t, stored.GPUDevicePath, retained.GPUDevicePath) } +func TestVGPURetentionWrapPending(t *testing.T) { + cause := errors.New("boot failed") + + retention := vgpuRetention{instanceID: "inst-1"} + assert.Same(t, cause, retention.wrapPending(cause)) + + retention.retained = true + pending := retention.wrapPending(cause) + var cleanupPending *VGPUCleanupPendingError + require.ErrorAs(t, pending, &cleanupPending) + assert.False(t, cleanupPending.Retained) + + retention.persisted = true + pending = retention.wrapPending(cause) + require.ErrorAs(t, pending, &cleanupPending) + assert.True(t, cleanupPending.Retained) +} + func TestVGPUCleanupPendingErrorUnwraps(t *testing.T) { t.Parallel() @@ -145,7 +176,7 @@ func TestVGPUDevicePendingCleanup(t *testing.T) { assert.Equal(t, device, *actual) assignedAt := time.Now().UTC() - retained := retainedVGPUFromCreateError(StoredMetadata{Id: "inst-1", Name: "named", Image: "img"}, assignedAt, wrapped) + retained := retainedVGPUFromCreateErrorForTest(StoredMetadata{Id: "inst-1", Name: "named", Image: "img"}, assignedAt, wrapped) require.NotNil(t, retained) assert.Equal(t, "inst-1", retained.Id) assert.Equal(t, "named", retained.Name, "identity fields must survive into the retention stub") @@ -157,7 +188,7 @@ func TestVGPUDevicePendingCleanup(t *testing.T) { actual, ok = vgpuDevicePendingCleanup(cause) assert.False(t, ok) assert.Nil(t, actual) - assert.Nil(t, retainedVGPUFromCreateError(StoredMetadata{Id: "inst-1"}, assignedAt, cause)) + assert.Nil(t, retainedVGPUFromCreateErrorForTest(StoredMetadata{Id: "inst-1"}, assignedAt, cause)) } type startRetentionNetworkManager struct { @@ -427,6 +458,7 @@ func TestCleanupStartVGPURestoresMetadataAfterBootFailure(t *testing.T) { exitCode := 1 rollbackMeta := metadata{StoredMetadata: StoredMetadata{ Id: id, + Name: "original name", GPUProfile: "NVIDIA L40S-2Q", Entrypoint: []string{"old-entrypoint"}, Cmd: []string{"old-command"}, @@ -437,6 +469,7 @@ func TestCleanupStartVGPURestoresMetadataAfterBootFailure(t *testing.T) { }} partial := rollbackMeta + partial.Name = "concurrent update" partial.Entrypoint = []string{"new-entrypoint"} partial.Cmd = []string{"new-command"} partial.StartedAt = ptr(time.Now().UTC()) @@ -455,6 +488,7 @@ func TestCleanupStartVGPURestoresMetadataAfterBootFailure(t *testing.T) { stored, err := m.loadMetadata(id) require.NoError(t, err) + assert.Equal(t, "concurrent update", stored.Name) assert.Equal(t, rollbackMeta.Entrypoint, stored.Entrypoint) assert.Equal(t, rollbackMeta.Cmd, stored.Cmd) assert.Equal(t, rollbackMeta.StartedAt, stored.StartedAt) From 61c563e6d3d8f106ec7144302420716f0b242dea Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Mon, 24 Aug 2026 14:54:32 +0000 Subject: [PATCH 56/76] Move vGPU reconciliation into instance manager --- cmd/api/main.go | 67 +------------------------- cmd/api/main_test.go | 41 ---------------- lib/builds/manager_test.go | 4 +- lib/devices/mdev_darwin.go | 2 +- lib/devices/vgpu_linux.go | 26 ++++++++-- lib/devices/vgpu_linux_test.go | 16 +++++++ lib/instances/manager.go | 1 + lib/instances/vgpu_reconcile.go | 61 ++++++++++++++++++++++++ lib/instances/vgpu_reconcile_test.go | 71 ++++++++++++++++++++++++++++ lib/instances/wait_test.go | 4 +- 10 files changed, 175 insertions(+), 118 deletions(-) create mode 100644 lib/instances/vgpu_reconcile.go create mode 100644 lib/instances/vgpu_reconcile_test.go diff --git a/cmd/api/main.go b/cmd/api/main.go index a4db0f3e9..da2748103 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -185,71 +185,6 @@ func configureUFFDGraduationController(cfg *config.Config, instanceManager insta }, logger), nil } -type vgpuReconcileInstanceLister interface { - ListInstancesForReconcile(context.Context) ([]instances.Instance, error) -} - -func liveInstanceVGPUDevicePaths(ctx context.Context, instanceManager instances.Manager) (map[string]struct{}, time.Duration, error) { - lister, ok := instanceManager.(vgpuReconcileInstanceLister) - if !ok { - return nil, 0, errors.New("instance manager does not support vGPU reconcile inventory") - } - allInstances, err := lister.ListInstancesForReconcile(ctx) - if err != nil { - return nil, 0, err - } - protected := make(map[string]struct{}) - var retryAfter time.Duration - for _, inst := range allInstances { - if inst.GPUDevicePath == "" { - continue - } - if inst.HypervisorPID != nil && instances.HypervisorMayBeAlive(inst.HypervisorProcessIdentity, inst.SocketPath) { - protected[inst.GPUDevicePath] = struct{}{} - continue - } - if inst.GPUAssignedAt == nil { - continue - } - remaining := instances.VGPUAssignmentStartupGracePeriod - time.Since(*inst.GPUAssignedAt) - if remaining <= 0 { - continue - } - protected[inst.GPUDevicePath] = struct{}{} - if retryAfter == 0 || remaining < retryAfter { - retryAfter = remaining - } - } - return protected, retryAfter, nil -} - -func reconcileVGPUs(ctx context.Context, instanceManager instances.Manager, logger *slog.Logger) { - protected, retryAfter, err := liveInstanceVGPUDevicePaths(ctx, instanceManager) - if err != nil { - // Operator-actionable: vendor VFIO reconciliation stays disabled - // host-wide (and releases fail closed on the same inventory) until - // the unreadable instance metadata is repaired. - logger.Error("failed to list instances for vGPU reconcile protection; reconciling mdev only", "error", err) - protected = nil - retryAfter = 0 - } - if err := devices.ReconcileVGPUs(ctx, protected); err != nil { - logger.Warn("failed to reconcile vGPU devices", "error", err) - } - if retryAfter <= 0 { - return - } - go func() { - timer := time.NewTimer(retryAfter) - defer timer.Stop() - select { - case <-ctx.Done(): - case <-timer.C: - reconcileVGPUs(ctx, instanceManager, logger) - } - }() -} - func run() error { startupStarted := time.Now() slog.Info("starting hypeman initialization") @@ -451,7 +386,7 @@ func run() error { // Reconcile vGPU devices (clears orphaned vGPUs from previous runs) logger.Info("Reconciling vGPU devices...") - reconcileVGPUs(ctx, app.InstanceManager, logger) + app.InstanceManager.ReconcileVGPUs(ctx) // Wire up resource validator for aggregate limit checking // This enables the instance manager to validate CPU, memory, network, and GPU diff --git a/cmd/api/main_test.go b/cmd/api/main_test.go index 0718b4cf3..b771e27fc 100644 --- a/cmd/api/main_test.go +++ b/cmd/api/main_test.go @@ -6,7 +6,6 @@ import ( "net/http" "net/http/httptest" "net/url" - "os/exec" "testing" "time" @@ -341,43 +340,3 @@ func TestImageNameWithSlashes_URLEncoding(t *testing.T) { }) } } - -type vgpuReconcileManagerStub struct { - instances.Manager - list []instances.Instance -} - -func (s vgpuReconcileManagerStub) ListInstancesForReconcile(context.Context) ([]instances.Instance, error) { - return s.list, nil -} - -func TestLiveInstanceVGPUDevicePathsRequiresReconcileInventory(t *testing.T) { - _, _, err := liveInstanceVGPUDevicePaths(context.Background(), struct{ instances.Manager }{}) - require.ErrorContains(t, err, "does not support vGPU reconcile inventory") -} - -func TestLiveInstanceVGPUDevicePathsBoundsStartupProtection(t *testing.T) { - dead := exec.Command("true") - require.NoError(t, dead.Run()) - deadPID := dead.Process.Pid - recent := time.Now().Add(-time.Minute) - stale := time.Now().Add(-instances.VGPUAssignmentStartupGracePeriod - time.Minute) - - manager := vgpuReconcileManagerStub{list: []instances.Instance{ - {StoredMetadata: instances.StoredMetadata{Id: "booting", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", GPUAssignedAt: &recent}}, - {StoredMetadata: instances.StoredMetadata{Id: "orphaned", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.5", GPUAssignedAt: &stale}}, - {StoredMetadata: instances.StoredMetadata{Id: "legacy", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.6"}}, - {StoredMetadata: instances.StoredMetadata{Id: "dead", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.7", HypervisorProcessIdentity: instances.HypervisorProcessIdentity{HypervisorPID: &deadPID}}}, - {StoredMetadata: instances.StoredMetadata{Id: "stale-pid-booting", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.8", HypervisorProcessIdentity: instances.HypervisorProcessIdentity{HypervisorPID: &deadPID}, GPUAssignedAt: &recent}}, - }} - - protected, retryAfter, err := liveInstanceVGPUDevicePaths(context.Background(), manager) - require.NoError(t, err) - require.Positive(t, retryAfter) - require.LessOrEqual(t, retryAfter, instances.VGPUAssignmentStartupGracePeriod) - assert.Contains(t, protected, "/sys/bus/pci/devices/0000:82:00.4") - assert.NotContains(t, protected, "/sys/bus/pci/devices/0000:82:00.5") - assert.NotContains(t, protected, "/sys/bus/pci/devices/0000:82:00.6") - assert.NotContains(t, protected, "/sys/bus/pci/devices/0000:82:00.7") - assert.Contains(t, protected, "/sys/bus/pci/devices/0000:82:00.8") -} diff --git a/lib/builds/manager_test.go b/lib/builds/manager_test.go index 44596bf68..ab390c72f 100644 --- a/lib/builds/manager_test.go +++ b/lib/builds/manager_test.go @@ -51,9 +51,7 @@ func (m *mockInstanceManager) ListInstances(ctx context.Context, filter *instanc return result, nil } -func (m *mockInstanceManager) ListInstancesForReconcile(ctx context.Context) ([]instances.Instance, error) { - return m.ListInstances(ctx, nil) -} +func (m *mockInstanceManager) ReconcileVGPUs(context.Context) {} func (m *mockInstanceManager) ListSnapshots(ctx context.Context, filter *instances.ListSnapshotsFilter) ([]instances.Snapshot, error) { return nil, nil diff --git a/lib/devices/mdev_darwin.go b/lib/devices/mdev_darwin.go index 4274063ed..4b726bb08 100644 --- a/lib/devices/mdev_darwin.go +++ b/lib/devices/mdev_darwin.go @@ -58,7 +58,7 @@ func DestroyVGPU(ctx context.Context, assignment VGPUAssignment) error { return nil } -func ReconcileVGPUs(ctx context.Context, protectedDevicePaths map[string]struct{}) error { +func ReconcileVGPUs(ctx context.Context, protectedDevicePaths map[string]struct{}, sweepVendorVFIO bool) error { return nil } diff --git a/lib/devices/vgpu_linux.go b/lib/devices/vgpu_linux.go index be92e8ee6..72827f7b2 100644 --- a/lib/devices/vgpu_linux.go +++ b/lib/devices/vgpu_linux.go @@ -105,20 +105,38 @@ func DestroyVGPU(ctx context.Context, assignment VGPUAssignment) error { } // ReconcileVGPUs releases orphaned vGPU assignments. -func ReconcileVGPUs(ctx context.Context, protectedDevicePaths map[string]struct{}) error { +func ReconcileVGPUs(ctx context.Context, protectedDevicePaths map[string]struct{}, sweepVendorVFIO bool) error { framework, _, err := DiscoverVGPU() if err != nil { return err } + return reconcileDiscoveredVGPUs( + ctx, + framework, + protectedDevicePaths, + sweepVendorVFIO, + func(ctx context.Context) error { return ReconcileMdevs(ctx, nil) }, + hostVendorVFIO.reconcile, + ) +} + +func reconcileDiscoveredVGPUs( + ctx context.Context, + framework VGPUFramework, + protectedDevicePaths map[string]struct{}, + sweepVendorVFIO bool, + reconcileMdev func(context.Context) error, + reconcileVendorVFIO func(context.Context, map[string]struct{}) error, +) error { switch framework { case VGPUFrameworkMdev: - return ReconcileMdevs(ctx, nil) + return reconcileMdev(ctx) case VGPUFrameworkVendorVFIO: - if protectedDevicePaths == nil { + if !sweepVendorVFIO { return nil } - return hostVendorVFIO.reconcile(ctx, protectedDevicePaths) + return reconcileVendorVFIO(ctx, protectedDevicePaths) default: return nil } diff --git a/lib/devices/vgpu_linux_test.go b/lib/devices/vgpu_linux_test.go index 7b03d9c26..32f9a30ea 100644 --- a/lib/devices/vgpu_linux_test.go +++ b/lib/devices/vgpu_linux_test.go @@ -3,6 +3,7 @@ package devices import ( + "context" "errors" "os" "path/filepath" @@ -12,6 +13,21 @@ import ( "github.com/stretchr/testify/require" ) +func TestReconcileDiscoveredVGPUsControlsVendorSweep(t *testing.T) { + protected := make(map[string]struct{}) + vendorCalls := 0 + reconcileVendor := func(context.Context, map[string]struct{}) error { + vendorCalls++ + return nil + } + + require.NoError(t, reconcileDiscoveredVGPUs(context.Background(), VGPUFrameworkVendorVFIO, protected, false, nil, reconcileVendor)) + assert.Zero(t, vendorCalls) + + require.NoError(t, reconcileDiscoveredVGPUs(context.Background(), VGPUFrameworkVendorVFIO, protected, true, nil, reconcileVendor)) + assert.Equal(t, 1, vendorCalls) +} + func TestDiscoverVGPUWithPropagatesMdevError(t *testing.T) { t.Parallel() diff --git a/lib/instances/manager.go b/lib/instances/manager.go index f3da80470..63e772c3e 100644 --- a/lib/instances/manager.go +++ b/lib/instances/manager.go @@ -29,6 +29,7 @@ import ( type Manager interface { ListInstances(ctx context.Context, filter *ListInstancesFilter) ([]Instance, error) + ReconcileVGPUs(ctx context.Context) ListSnapshots(ctx context.Context, filter *ListSnapshotsFilter) ([]Snapshot, error) GetSnapshot(ctx context.Context, snapshotID string) (*Snapshot, error) CreateInstance(ctx context.Context, req CreateInstanceRequest) (*Instance, error) diff --git a/lib/instances/vgpu_reconcile.go b/lib/instances/vgpu_reconcile.go new file mode 100644 index 000000000..85d3a6e0d --- /dev/null +++ b/lib/instances/vgpu_reconcile.go @@ -0,0 +1,61 @@ +package instances + +import ( + "context" + "time" + + "github.com/kernel/hypeman/lib/devices" + "github.com/kernel/hypeman/lib/logger" +) + +func (m *manager) liveVGPUReconcileProtection(ctx context.Context) (map[string]struct{}, time.Duration, error) { + allInstances, err := m.ListInstancesForReconcile(ctx) + if err != nil { + return nil, 0, err + } + protected := make(map[string]struct{}) + var retryAfter time.Duration + for i := range allInstances { + stored := &allInstances[i].StoredMetadata + if stored.GPUDevicePath == "" { + continue + } + livePID := stored.HypervisorPID != nil && HypervisorMayBeAlive(stored.HypervisorProcessIdentity, stored.SocketPath) + live, remaining := vgpuAssignmentLiveness(stored, m.nowUTC(), livePID) + if !live { + continue + } + protected[stored.GPUDevicePath] = struct{}{} + if remaining > 0 && (retryAfter == 0 || remaining < retryAfter) { + retryAfter = remaining + } + } + return protected, retryAfter, nil +} + +// ReconcileVGPUs releases orphaned vGPU assignments. +func (m *manager) ReconcileVGPUs(ctx context.Context) { + log := logger.FromContext(ctx) + protected, retryAfter, err := m.liveVGPUReconcileProtection(ctx) + sweepVendorVFIO := err == nil + if err != nil { + log.ErrorContext(ctx, "failed to list instances for vGPU reconcile protection; skipping vendor VFIO reconcile, mdev reconcile still runs", "error", err) + protected = make(map[string]struct{}) + retryAfter = 0 + } + if err := devices.ReconcileVGPUs(ctx, protected, sweepVendorVFIO); err != nil { + log.WarnContext(ctx, "failed to reconcile vGPU devices", "error", err) + } + if retryAfter <= 0 { + return + } + go func() { + timer := time.NewTimer(retryAfter) + defer timer.Stop() + select { + case <-ctx.Done(): + case <-timer.C: + m.ReconcileVGPUs(ctx) + } + }() +} diff --git a/lib/instances/vgpu_reconcile_test.go b/lib/instances/vgpu_reconcile_test.go new file mode 100644 index 000000000..1fbd90384 --- /dev/null +++ b/lib/instances/vgpu_reconcile_test.go @@ -0,0 +1,71 @@ +package instances + +import ( + "os/exec" + "testing" + "time" + + "github.com/kernel/hypeman/lib/paths" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestLiveVGPUReconcileProtectionBoundsStartupProtection(t *testing.T) { + dead := exec.Command("true") + require.NoError(t, dead.Run()) + deadPID := dead.Process.Pid + now := time.Now().UTC() + recent := now.Add(-time.Minute) + stale := now.Add(-VGPUAssignmentStartupGracePeriod - time.Minute) + + m := &manager{paths: paths.New(t.TempDir()), now: func() time.Time { return now }} + instances := []StoredMetadata{ + {Id: "booting", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", GPUAssignedAt: &recent}, + {Id: "orphaned", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.5", GPUAssignedAt: &stale}, + {Id: "legacy", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.6"}, + {Id: "dead", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.7", HypervisorProcessIdentity: HypervisorProcessIdentity{HypervisorPID: &deadPID}}, + {Id: "stale-pid-booting", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.8", HypervisorProcessIdentity: HypervisorProcessIdentity{HypervisorPID: &deadPID}, GPUAssignedAt: &recent}, + } + for i := range instances { + require.NoError(t, m.ensureDirectories(instances[i].Id)) + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: instances[i]})) + } + + protected, retryAfter, err := m.liveVGPUReconcileProtection(t.Context()) + require.NoError(t, err) + assert.Equal(t, VGPUAssignmentStartupGracePeriod-time.Minute, retryAfter) + assert.Contains(t, protected, "/sys/bus/pci/devices/0000:82:00.4") + assert.NotContains(t, protected, "/sys/bus/pci/devices/0000:82:00.5") + assert.NotContains(t, protected, "/sys/bus/pci/devices/0000:82:00.6") + assert.NotContains(t, protected, "/sys/bus/pci/devices/0000:82:00.7") + assert.Contains(t, protected, "/sys/bus/pci/devices/0000:82:00.8") +} + +func TestVGPUAssignmentLiveness(t *testing.T) { + now := time.Now().UTC() + recent := now.Add(-time.Minute) + stale := now.Add(-VGPUAssignmentStartupGracePeriod - time.Minute) + pid := 123 + + tests := []struct { + name string + stored StoredMetadata + livePID bool + live bool + remaining time.Duration + }{ + {name: "live PID", stored: StoredMetadata{HypervisorProcessIdentity: HypervisorProcessIdentity{HypervisorPID: &pid}}, livePID: true, live: true}, + {name: "dead PID recent assignment", stored: StoredMetadata{HypervisorProcessIdentity: HypervisorProcessIdentity{HypervisorPID: &pid}, GPUAssignedAt: &recent}, live: true, remaining: VGPUAssignmentStartupGracePeriod - time.Minute}, + {name: "dead PID stale assignment", stored: StoredMetadata{HypervisorProcessIdentity: HypervisorProcessIdentity{HypervisorPID: &pid}, GPUAssignedAt: &stale}}, + {name: "no PID recent assignment", stored: StoredMetadata{GPUAssignedAt: &recent}, live: true, remaining: VGPUAssignmentStartupGracePeriod - time.Minute}, + {name: "no PID stale assignment", stored: StoredMetadata{GPUAssignedAt: &stale}}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + live, remaining := vgpuAssignmentLiveness(&tt.stored, now, tt.livePID) + assert.Equal(t, tt.live, live) + assert.Equal(t, tt.remaining, remaining) + }) + } +} diff --git a/lib/instances/wait_test.go b/lib/instances/wait_test.go index 415003594..3ab02e402 100644 --- a/lib/instances/wait_test.go +++ b/lib/instances/wait_test.go @@ -32,9 +32,7 @@ func (s *stubManager) GetInstance(ctx context.Context, id string) (*Instance, er func (s *stubManager) ListInstances(context.Context, *ListInstancesFilter) ([]Instance, error) { return nil, nil } -func (s *stubManager) ListInstancesForReconcile(context.Context) ([]Instance, error) { - return nil, nil -} +func (s *stubManager) ReconcileVGPUs(context.Context) {} func (s *stubManager) ListSnapshots(context.Context, *ListSnapshotsFilter) ([]Snapshot, error) { return nil, nil } From e417c85792748e28b38243009a17a4a0b7c86452 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Mon, 24 Aug 2026 14:54:32 +0000 Subject: [PATCH 57/76] Derive QEMU VFIO grace from VM config --- cmd/api/main_test.go | 2 -- lib/hypervisor/qemu/process.go | 28 +++++++++------------------- lib/hypervisor/qemu/process_test.go | 17 +++++++++++++++++ lib/hypervisor/vfio.go | 7 +++++++ lib/instances/process_identity.go | 8 +------- 5 files changed, 34 insertions(+), 28 deletions(-) create mode 100644 lib/hypervisor/vfio.go diff --git a/cmd/api/main_test.go b/cmd/api/main_test.go index b771e27fc..34dbba428 100644 --- a/cmd/api/main_test.go +++ b/cmd/api/main_test.go @@ -2,7 +2,6 @@ package main import ( "bytes" - "context" "net/http" "net/http/httptest" "net/url" @@ -12,7 +11,6 @@ import ( "github.com/getkin/kin-openapi/openapi3filter" "github.com/go-chi/chi/v5" "github.com/golang-jwt/jwt/v5" - "github.com/kernel/hypeman/lib/instances" mw "github.com/kernel/hypeman/lib/middleware" "github.com/kernel/hypeman/lib/oapi" nethttpmiddleware "github.com/oapi-codegen/nethttp-middleware" diff --git a/lib/hypervisor/qemu/process.go b/lib/hypervisor/qemu/process.go index cddd51824..367bf14e3 100644 --- a/lib/hypervisor/qemu/process.go +++ b/lib/hypervisor/qemu/process.go @@ -40,12 +40,6 @@ const ( // socketDialTimeout is timeout for individual socket connection attempts socketDialTimeout = 100 * time.Millisecond - // vfioTermGrace is how long start-failure cleanup waits for a - // VFIO-attached QEMU to exit on SIGTERM before SIGKILL. Only failed - // starts pay it, and only when the process ignores SIGTERM; observed - // mid-init VFIO teardown takes 1-2s. - vfioTermGrace = 5 * time.Second - // clientCreateTimeout is how long to retry QMP client creation after the // socket appears. Under high parallel load the socket can accept connections // slightly later than file creation/availability. @@ -320,20 +314,17 @@ func (p *startedProcess) cleanup() { _ = os.Remove(p.socketPath) } -// hasVFIODevice reports whether the QEMU command line attaches a VFIO device. -func hasVFIODevice(args []string) bool { - for _, arg := range args { - if strings.Contains(arg, "vfio-pci") { - return true - } +func vfioTermGraceFor(cfg hypervisor.VMConfig) time.Duration { + if cfg.VGPUDevicePath != "" || len(cfg.PCIDevices) > 0 { + return hypervisor.VFIOTermGrace } - return false + return 0 } // startQEMUProcess handles the common QEMU process startup logic. // Returns the PID, hypervisor client, and a cleanup function. // The cleanup function must be called on error; call cleanup.Release() on success. -func (s *Starter) startQEMUProcess(ctx context.Context, p *paths.Paths, version string, socketPath string, args []string) (int, *QEMU, *cleanup.Cleanup, error) { +func (s *Starter) startQEMUProcess(ctx context.Context, p *paths.Paths, version string, socketPath string, args []string, termGrace time.Duration) (int, *QEMU, *cleanup.Cleanup, error) { log := logger.FromContext(ctx) processAttrs := hypervisor.TraceAttributesFromContext(ctx) processAttrs = append(processAttrs, @@ -403,9 +394,8 @@ func (s *Starter) startQEMUProcess(ctx context.Context, p *paths.Paths, version } pid := proc.pid - if hasVFIODevice(args) { - proc.termGrace = vfioTermGrace - } + // Only failed starts pay the VFIO termination grace. + proc.termGrace = termGrace log.DebugContext(processCtx, "QEMU process started", "pid", pid, "duration_ms", time.Since(processStartTime).Milliseconds()) // Setup cleanup to kill, reap, and remove the socket if subsequent steps fail. @@ -522,7 +512,7 @@ func (s *Starter) StartVM(ctx context.Context, p *paths.Paths, version string, s // Build command arguments: QMP socket + VM configuration args := buildQMPArgs(socketPath) args = append(args, buildArgs(attempt, machineType)...) - pid, hv, cu, err = s.startQEMUProcess(ctx, p, version, socketPath, args) + pid, hv, cu, err = s.startQEMUProcess(ctx, p, version, socketPath, args, vfioTermGraceFor(attempt)) if err == nil { booted = attempt started = true @@ -657,7 +647,7 @@ func (s *Starter) RestoreVM(ctx context.Context, p *paths.Paths, version string, incomingURI := "exec:cat < " + memoryFile args = append(args, "-incoming", incomingURI) - pid, hv, cu, err := s.startQEMUProcess(ctx, p, version, socketPath, args) + pid, hv, cu, err := s.startQEMUProcess(ctx, p, version, socketPath, args, vfioTermGraceFor(config)) if err != nil { return 0, nil, err } diff --git a/lib/hypervisor/qemu/process_test.go b/lib/hypervisor/qemu/process_test.go index 2bdf27d4f..ad08aea24 100644 --- a/lib/hypervisor/qemu/process_test.go +++ b/lib/hypervisor/qemu/process_test.go @@ -411,6 +411,23 @@ func TestWaitForSocketOrExitReturnsEarlyWhenProcessDies(t *testing.T) { assert.True(t, cmd.ProcessState.Exited()) } +func TestVFIOTermGraceFor(t *testing.T) { + tests := []struct { + name string + cfg hypervisor.VMConfig + want time.Duration + }{ + {name: "vGPU", cfg: hypervisor.VMConfig{VGPUDevicePath: "/sys/bus/mdev/devices/test"}, want: hypervisor.VFIOTermGrace}, + {name: "PCI device", cfg: hypervisor.VMConfig{PCIDevices: []string{"0000:01:00.0"}}, want: hypervisor.VFIOTermGrace}, + {name: "no VFIO device", cfg: hypervisor.VMConfig{}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, vfioTermGraceFor(tt.cfg)) + }) + } +} + func TestCleanupSIGTERMsProcessWithTermGrace(t *testing.T) { socketPath := filepath.Join(t.TempDir(), "qemu.sock") markerPath := filepath.Join(t.TempDir(), "terminated") diff --git a/lib/hypervisor/vfio.go b/lib/hypervisor/vfio.go new file mode 100644 index 000000000..1744cfc84 --- /dev/null +++ b/lib/hypervisor/vfio.go @@ -0,0 +1,7 @@ +package hypervisor + +import "time" + +// VFIOTermGrace allows VFIO teardown to finish after SIGTERM before SIGKILL; +// SIGKILL during initialization can wedge the VF, while teardown takes 1-2s. +const VFIOTermGrace = 5 * time.Second diff --git a/lib/instances/process_identity.go b/lib/instances/process_identity.go index 0801aca6c..83aa682bc 100644 --- a/lib/instances/process_identity.go +++ b/lib/instances/process_identity.go @@ -27,19 +27,13 @@ const linuxBootIDPath = "/proc/sys/kernel/random/boot_id" // does not unstick it, so the wait is short to keep stop and delete fast. const hypervisorSIGKILLWaitTimeout = 2 * time.Second -// defaultVGPUInitTermGrace is how long terminateThenKill waits for a vGPU -// hypervisor to exit on SIGTERM before SIGKILL. Only force-kill paths pay it, -// only for vGPU instances, and only when the process ignores SIGTERM; -// observed mid-init VFIO teardown takes 1-2s. -const defaultVGPUInitTermGrace = 5 * time.Second - // vgpuTermGrace returns the SIGTERM wait used before hard-killing a vGPU // hypervisor. func (m *manager) vgpuTermGrace() time.Duration { if m.vgpuInitTermGrace > 0 { return m.vgpuInitTermGrace } - return defaultVGPUInitTermGrace + return hypervisor.VFIOTermGrace } // terminateThenKill hard-kills the hypervisor process, first giving any vGPU From 648373ddaab7a266f824d4dd6fbe305ba0d961bd Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Mon, 24 Aug 2026 15:25:09 +0000 Subject: [PATCH 58/76] Tighten vGPU lifecycle comments --- lib/hypervisor/qemu/process.go | 6 ++---- lib/instances/manager.go | 11 ++++------- lib/instances/metrics.go | 5 ++--- lib/instances/process_identity.go | 18 +++++++----------- lib/instances/storage.go | 5 ++--- lib/instances/vgpu.go | 24 +++++++++--------------- lib/instances/vgpu_orphan.go | 20 +++++++++----------- 7 files changed, 35 insertions(+), 54 deletions(-) diff --git a/lib/hypervisor/qemu/process.go b/lib/hypervisor/qemu/process.go index 367bf14e3..de560c10d 100644 --- a/lib/hypervisor/qemu/process.go +++ b/lib/hypervisor/qemu/process.go @@ -228,10 +228,8 @@ type startedProcess struct { pid int socketPath string // termGrace, when non-zero, makes cleanup send SIGTERM and wait this long - // before SIGKILL. Set for VFIO-attached processes: hard-killing QEMU while - // the NVIDIA vGPU plugin is initializing can silently wedge the VF until - // its parent GPU's SR-IOV is cycled, while a terminating QEMU runs its - // device teardown and leaves the VF reusable. + // before SIGKILL. Set for VFIO-attached processes: SIGKILL during vGPU + // plugin init can wedge the VF until its parent GPU is SR-IOV cycled. termGrace time.Duration waitDone chan error waitConsumed bool diff --git a/lib/instances/manager.go b/lib/instances/manager.go index 63e772c3e..3954af19f 100644 --- a/lib/instances/manager.go +++ b/lib/instances/manager.go @@ -752,9 +752,8 @@ func (m *manager) DefaultHypervisor() hypervisor.Type { } // ListInstancesForReconcile returns every instance's stored metadata or an -// invalid metadata error. It does not derive state: reconcile protection only -// needs raw metadata fields, and hydration would query the hypervisor of -// every instance on the host before the API serves. +// invalid metadata error. It does not derive state: hydration would query +// every hypervisor on the host before the API serves. func (m *manager) ListInstancesForReconcile(ctx context.Context) ([]Instance, error) { files, err := m.listMetadataFilesStrict() if err != nil { @@ -766,10 +765,8 @@ func (m *manager) ListInstancesForReconcile(ctx context.Context) ([]Instance, er meta, err := m.loadMetadata(id) if err != nil { if errors.Is(err, ErrNotFound) { - // Deleted between listing and load; a vanished record cannot - // claim a VF. Failing here instead would zero the grace-period - // retry and disable the vendor VFIO sweep whenever it races a - // concurrent delete. + // Deleted between listing and load; failing instead would + // disable the vendor VFIO sweep whenever it races a delete. continue } return nil, fmt.Errorf("load metadata for instance %s: %w", id, err) diff --git a/lib/instances/metrics.go b/lib/instances/metrics.go index bdffbdbce..085216fdf 100644 --- a/lib/instances/metrics.go +++ b/lib/instances/metrics.go @@ -574,9 +574,8 @@ func (m *manager) recordTimeToRunning(ctx context.Context, stored *StoredMetadat } // recordVGPUOrphanReleaseAbandoned records an orphaned vGPU release retry -// loop giving up: the VF stays allocated (capacity silently reduced) until -// startup reconciliation or manual remediation, so it must be visible beyond -// a log line. +// loop giving up: the VF stays allocated until startup reconciliation or +// manual remediation, so it must be visible beyond a log line. func (m *manager) recordVGPUOrphanReleaseAbandoned(ctx context.Context) { if m.metrics == nil { return diff --git a/lib/instances/process_identity.go b/lib/instances/process_identity.go index 83aa682bc..dd546c420 100644 --- a/lib/instances/process_identity.go +++ b/lib/instances/process_identity.go @@ -37,13 +37,10 @@ func (m *manager) vgpuTermGrace() time.Duration { } // terminateThenKill hard-kills the hypervisor process, first giving any vGPU -// instance a SIGTERM grace: SIGKILL during guest driver init can silently -// wedge the VF until its parent GPU's SR-IOV is cycled (see -// lib/devices/GPU.md), while a terminating QEMU runs its VFIO teardown. The -// grace applies in every state, not just Initializing, because the instance -// reports Running seconds before the guest driver finishes initializing and -// nothing host-side observes that boundary; post-init the SIGTERM is proven -// harmless and costs the grace only when the process ignores it. +// instance a SIGTERM grace: SIGKILL during guest driver init can wedge the VF +// until its parent GPU is SR-IOV cycled (see lib/devices/GPU.md). The grace +// applies in every state because the instance reports Running seconds before +// driver init finishes and nothing host-side observes that boundary. func (m *manager) terminateThenKill(ctx context.Context, inst *Instance, pid int) error { if inst.GPUProfile != "" { if syscall.Kill(pid, syscall.SIGTERM) == nil && WaitForProcessExit(pid, m.vgpuTermGrace()) { @@ -211,10 +208,9 @@ func classifyResolvedHypervisorOwner(socketPath string, stored, resolved int, er } // HypervisorMayBeAlive reports whether the recorded hypervisor process may -// still be running. It fails open: when ownership cannot be resolved it -// returns true, which is the safe direction for its callers (reconcile -// protection and claim checks, where true means "protect"). Do not use it to -// authorize teardown. +// still be running. It fails open (unresolvable ownership returns true, the +// safe direction for reconcile protection and claim checks); do not use it +// to authorize teardown. func HypervisorMayBeAlive(id HypervisorProcessIdentity, socketPath string) bool { pid, err := resolveLiveHypervisorPID(id, socketPath) return err != nil || pid > 0 diff --git a/lib/instances/storage.go b/lib/instances/storage.go index 6a2354624..40bbba684 100644 --- a/lib/instances/storage.go +++ b/lib/instances/storage.go @@ -194,9 +194,8 @@ func (m *manager) listMetadataFiles() ([]string, error) { } // listMetadataFilesStrict returns paths to all instance metadata files, -// failing on any stat error other than absence. Fail-closed callers (the -// vGPU release claim scan and startup reconcile protection) use it so an -// unreadable instance is an error instead of silently missing. +// failing on any stat error other than absence, so fail-closed callers see +// an unreadable instance as an error instead of silently missing. func (m *manager) listMetadataFilesStrict() ([]string, error) { return m.walkMetadataFiles(true) } diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index 3561cbc5a..8ffe64dd8 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -123,10 +123,8 @@ func (m *manager) cleanupStartVGPU(ctx context.Context, instanceID string, devic if !retained { return false, false } - // The mid-start save may already have persisted this assignment, in - // which case the on-disk record still points at the device and - // delete or a retried start can release it (matching create's - // retention-survives check). + // The mid-start save may already have persisted this assignment, so + // delete or a retried start can still release it. if meta, loadErr := m.loadMetadata(instanceID); loadErr == nil && storedVGPUDevicePath(&meta.StoredMetadata) == device.SysfsPath { return true, true } @@ -159,11 +157,10 @@ func restoreStartMutatedFields(dst, src *StoredMetadata) { func (m *manager) releaseStoredVGPU(ctx context.Context, stored *StoredMetadata) error { path := storedVGPUDevicePath(stored) if path != "" { - // Vendor VFIO VFs are reused across instances, so stale metadata can - // point at a path claimed by a live instance and the release must fail - // closed on an incomplete inventory. mdev UUIDs are unique and never - // reused, so skip the scan there — it would let one unreadable - // metadata file block every mdev release on the host. + // Vendor VFIO VFs are reused across instances, so the release must + // fail closed on an incomplete inventory. mdev UUIDs are never reused; + // scanning there would let one unreadable metadata file block every + // mdev release on the host. claimed := false if stored.GPUFramework == devices.VGPUFrameworkVendorVFIO { var err error @@ -192,12 +189,9 @@ func (m *manager) releaseStoredVGPU(ctx context.Context, stored *StoredMetadata) } // vgpuAssignmentClaimedByLiveInstance reports whether another live instance's -// stored metadata claims devicePath. It reads raw metadata instead of -// hydrating full instances: the scan runs on every vendor VFIO release, and -// deriving state would query the hypervisor of every instance on the host. -// A confirmed live claimant returns true. Unreadable metadata, a recent -// assignment without a PID, or unverifiable process ownership returns an error -// so the requester retains its assignment for a later retry. +// stored metadata claims devicePath. Unreadable metadata, a recent assignment +// without a PID, or unverifiable process ownership returns an error so the +// requester retains its assignment for a later retry. func (m *manager) vgpuAssignmentClaimedByLiveInstance(ctx context.Context, excludeID, devicePath string) (bool, error) { files, err := m.listMetadataFilesStrict() if err != nil { diff --git a/lib/instances/vgpu_orphan.go b/lib/instances/vgpu_orphan.go index b3ff2b33b..d85a68bf3 100644 --- a/lib/instances/vgpu_orphan.go +++ b/lib/instances/vgpu_orphan.go @@ -8,22 +8,20 @@ import ( ) const ( - // orphanedVGPUReleaseMaxAttempts bounds the retry loop so a genuinely - // wedged VF degrades to one operator-actionable error instead of - // indefinite log churn. At the default interval this covers ten minutes, - // far beyond the seconds a dying VMM normally needs to finish kernel-side - // VFIO teardown. + // Bounds the retry loop (~10 minutes at the default interval, far beyond + // normal VFIO teardown) so a wedged VF degrades to one operator-actionable + // error instead of indefinite log churn. orphanedVGPUReleaseMaxAttempts = 20 defaultOrphanedVGPUReleaseRetryDelay = 30 * time.Second ) // scheduleOrphanedVGPURelease retries a vGPU release that failed during a -// completed delete, off the request path. A GPU-busy VMM routinely outlives -// delete's force-kill wait while the kernel finishes VFIO teardown, and once -// the metadata is deleted nothing else releases the VF until the next -// startup reconciliation. Each attempt re-runs releaseStoredVGPU, so the -// claim scan and destroy guards apply on every retry. The queue is in-memory -// only: a restart abandons it and startup reconciliation sweeps the VF. +// completed delete, off the request path: a GPU-busy VMM routinely outlives +// delete's force-kill wait, and once metadata is deleted nothing else +// releases the VF until startup reconciliation. Each attempt re-runs +// releaseStoredVGPU, so the claim scan and destroy guards apply on every +// retry. The queue is in-memory only; a restart abandons it and startup +// reconciliation sweeps the VF. func (m *manager) scheduleOrphanedVGPURelease(ctx context.Context, stored StoredMetadata) { path := storedVGPUDevicePath(&stored) if path == "" { From 928d61cec021cb9d09ec4f877dd4d7d9b067f609 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Mon, 24 Aug 2026 16:25:12 +0000 Subject: [PATCH 59/76] Retry vGPU recovery paths that previously waited for restart - Retry the vendor VFIO reconcile sweep with a bounded delay when the startup instance listing fails, instead of disabling orphan recovery until the next process restart. One pending retry at a time. - Schedule the in-process orphaned-release retry when a rollback's retention record cannot be saved (create and start), instead of leaking the VF until restart. The retry scans claims without a self-exclusion because a restarted instance may hold the same VF. - Reject snapshot restore into a vGPU retention stub, matching start, fork, and snapshot. - Give passthrough PCI instances the same SIGTERM grace as vGPU instances on stop/delete, matching the QEMU-side vfioTermGraceFor. - Render the vgpu_cleanup_pending API detail from the error itself instead of duplicating its prose; use the manager clock in the claim scan; collapse the create-rollback retention branch. - Move ReconcileVGPUs off the Manager interface to a startup type assertion and unexport listInstancesForReconcile and hypervisorMayBeAlive. --- cmd/api/api/instances.go | 10 ++++---- cmd/api/main.go | 9 +++++-- lib/builds/manager_test.go | 2 -- lib/instances/create.go | 8 ++---- lib/instances/manager.go | 12 ++++++--- lib/instances/process_identity.go | 20 ++++++++------- lib/instances/query_test.go | 6 ++--- lib/instances/snapshot.go | 5 ++++ lib/instances/snapshot_test.go | 37 ++++++++++++++++++++++++++++ lib/instances/start.go | 4 +++ lib/instances/vgpu.go | 21 +++++++++++++--- lib/instances/vgpu_orphan.go | 17 +++++++------ lib/instances/vgpu_reconcile.go | 21 +++++++++++++--- lib/instances/vgpu_reconcile_test.go | 29 ++++++++++++++++++++++ lib/instances/vgpu_retention.go | 8 +++--- lib/instances/vgpu_test.go | 15 ++++++++--- lib/instances/wait_test.go | 1 - 17 files changed, 172 insertions(+), 53 deletions(-) diff --git a/cmd/api/api/instances.go b/cmd/api/api/instances.go index 1c502c928..46be5cec4 100644 --- a/cmd/api/api/instances.go +++ b/cmd/api/api/instances.go @@ -439,11 +439,11 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst // and inner error detail shared by the create and start handlers. The // retained guidance names the verb-specific way to release the assignment. func vgpuCleanupPendingDetail(pending *instances.VGPUCleanupPendingError, action, retainedGuidance string) (string, *oapi.ErrorDetail) { - message := fmt.Sprintf("failed to %s instance: %v; vGPU release failed during rollback and instance %s retains the assignment, %s", action, pending.Err, pending.InstanceID, retainedGuidance) - innerCode := "vgpu_retained_instance" - if !pending.Retained { - message = fmt.Sprintf("failed to %s instance: %v; vGPU release failed during rollback and the retention record for instance %s could not be saved; the assignment is recovered on the next startup reconcile", action, pending.Err, pending.InstanceID) - innerCode = "vgpu_unretained_instance" + message := fmt.Sprintf("failed to %s instance: %v", action, pending) + innerCode := "vgpu_unretained_instance" + if pending.Retained { + message += "; " + retainedGuidance + innerCode = "vgpu_retained_instance" } return message, &oapi.ErrorDetail{ Code: lo.ToPtr(innerCode), diff --git a/cmd/api/main.go b/cmd/api/main.go index da2748103..20ac8dc60 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -384,9 +384,14 @@ func run() error { return fmt.Errorf("reconcile device state: %w", err) } - // Reconcile vGPU devices (clears orphaned vGPUs from previous runs) + // Reconcile vGPU devices (clears orphaned vGPUs from previous runs). + // Type-asserted rather than added to instances.Manager so alternate + // Manager implementations compiled against the public module keep + // building without this startup-only method. logger.Info("Reconciling vGPU devices...") - app.InstanceManager.ReconcileVGPUs(ctx) + if r, ok := app.InstanceManager.(interface{ ReconcileVGPUs(context.Context) }); ok { + r.ReconcileVGPUs(ctx) + } // Wire up resource validator for aggregate limit checking // This enables the instance manager to validate CPU, memory, network, and GPU diff --git a/lib/builds/manager_test.go b/lib/builds/manager_test.go index ab390c72f..a137edc66 100644 --- a/lib/builds/manager_test.go +++ b/lib/builds/manager_test.go @@ -51,8 +51,6 @@ func (m *mockInstanceManager) ListInstances(ctx context.Context, filter *instanc return result, nil } -func (m *mockInstanceManager) ReconcileVGPUs(context.Context) {} - func (m *mockInstanceManager) ListSnapshots(ctx context.Context, filter *instances.ListSnapshotsFilter) ([]instances.Snapshot, error) { return nil, nil } diff --git a/lib/instances/create.go b/lib/instances/create.go index f4ece0100..de301a95d 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -278,7 +278,6 @@ func (m *manager) createInstance( var gpuDevicePath string var gpuMdevUUID string var gpuAssignedAt *time.Time - var stored *StoredMetadata retention := vgpuRetention{instanceID: id} // Setup cleanup stack early so device attachment errors trigger cleanup. @@ -344,10 +343,7 @@ func (m *manager) createInstance( } if err := m.destroyVGPUAssignment(ctx, assignment); err != nil { log.WarnContext(ctx, "failed to destroy vGPU on cleanup", "instance_id", id, "uuid", gpuDevice.MdevUUID, "error", err) - retention.retain(stored) - if stored == nil { - retention.retainFromDevice(retentionStub(), gpuDevice, *gpuAssignedAt) - } + retention.retainFromDevice(retentionStub(), gpuDevice, *gpuAssignedAt) } }) } @@ -388,7 +384,7 @@ func (m *manager) createInstance( if err != nil { return nil, err } - stored = &StoredMetadata{ + stored := &StoredMetadata{ Id: id, Name: req.Name, Image: req.Image, diff --git a/lib/instances/manager.go b/lib/instances/manager.go index 3954af19f..da1fe59d3 100644 --- a/lib/instances/manager.go +++ b/lib/instances/manager.go @@ -8,6 +8,7 @@ import ( "path/filepath" "strings" "sync" + "sync/atomic" "time" "github.com/kernel/hypeman/lib/devices" @@ -29,7 +30,6 @@ import ( type Manager interface { ListInstances(ctx context.Context, filter *ListInstancesFilter) ([]Instance, error) - ReconcileVGPUs(ctx context.Context) ListSnapshots(ctx context.Context, filter *ListSnapshotsFilter) ([]Snapshot, error) GetSnapshot(ctx context.Context, snapshotID string) (*Snapshot, error) CreateInstance(ctx context.Context, req CreateInstanceRequest) (*Instance, error) @@ -222,6 +222,12 @@ type manager struct { orphanedVGPUs map[string]struct{} orphanedVGPURetryDelay time.Duration + // One pending vGPU reconcile retry at a time, for both the startup-grace + // and listing-failure paths. vgpuReconcileRetryDelay overrides the + // listing-failure delay in tests; zero means the default. + vgpuReconcileRetryPending atomic.Bool + vgpuReconcileRetryDelay time.Duration + // vgpuInitTermGrace overrides terminateThenKill's SIGTERM wait for vGPU // instances still initializing; zero means the default. vgpuInitTermGrace time.Duration @@ -751,10 +757,10 @@ func (m *manager) DefaultHypervisor() hypervisor.Type { return m.defaultHypervisor } -// ListInstancesForReconcile returns every instance's stored metadata or an +// listInstancesForReconcile returns every instance's stored metadata or an // invalid metadata error. It does not derive state: hydration would query // every hypervisor on the host before the API serves. -func (m *manager) ListInstancesForReconcile(ctx context.Context) ([]Instance, error) { +func (m *manager) listInstancesForReconcile(ctx context.Context) ([]Instance, error) { files, err := m.listMetadataFilesStrict() if err != nil { return nil, err diff --git a/lib/instances/process_identity.go b/lib/instances/process_identity.go index dd546c420..cd8ff2fe1 100644 --- a/lib/instances/process_identity.go +++ b/lib/instances/process_identity.go @@ -36,17 +36,19 @@ func (m *manager) vgpuTermGrace() time.Duration { return hypervisor.VFIOTermGrace } -// terminateThenKill hard-kills the hypervisor process, first giving any vGPU -// instance a SIGTERM grace: SIGKILL during guest driver init can wedge the VF -// until its parent GPU is SR-IOV cycled (see lib/devices/GPU.md). The grace -// applies in every state because the instance reports Running seconds before -// driver init finishes and nothing host-side observes that boundary. +// terminateThenKill hard-kills the hypervisor process, first giving any +// instance with VFIO devices (a vGPU VF or passthrough PCI devices, matching +// the QEMU-side vfioTermGraceFor) a SIGTERM grace: SIGKILL during guest +// driver init can wedge the device until its parent GPU is SR-IOV cycled +// (see lib/devices/GPU.md). The grace applies in every state because the +// instance reports Running seconds before driver init finishes and nothing +// host-side observes that boundary. func (m *manager) terminateThenKill(ctx context.Context, inst *Instance, pid int) error { - if inst.GPUProfile != "" { + if inst.GPUProfile != "" || len(inst.Devices) > 0 { if syscall.Kill(pid, syscall.SIGTERM) == nil && WaitForProcessExit(pid, m.vgpuTermGrace()) { return nil } - logger.FromContext(ctx).WarnContext(ctx, "vGPU hypervisor did not exit on SIGTERM; hard-killing, VF may wedge if the guest driver was initializing", + logger.FromContext(ctx).WarnContext(ctx, "hypervisor with VFIO devices did not exit on SIGTERM; hard-killing, device may wedge if the guest driver was initializing", "instance_id", inst.Id, "device_path", inst.GPUDevicePath) } return killProcessAndWait(pid) @@ -207,11 +209,11 @@ func classifyResolvedHypervisorOwner(socketPath string, stored, resolved int, er return 0, fmt.Errorf("cannot confirm ownership of socket %s: %w", socketPath, err) } -// HypervisorMayBeAlive reports whether the recorded hypervisor process may +// hypervisorMayBeAlive reports whether the recorded hypervisor process may // still be running. It fails open (unresolvable ownership returns true, the // safe direction for reconcile protection and claim checks); do not use it // to authorize teardown. -func HypervisorMayBeAlive(id HypervisorProcessIdentity, socketPath string) bool { +func hypervisorMayBeAlive(id HypervisorProcessIdentity, socketPath string) bool { pid, err := resolveLiveHypervisorPID(id, socketPath) return err != nil || pid > 0 } diff --git a/lib/instances/query_test.go b/lib/instances/query_test.go index b3dbfba41..3d12c739d 100644 --- a/lib/instances/query_test.go +++ b/lib/instances/query_test.go @@ -31,12 +31,12 @@ func TestListInstancesForReconcileFailsOnInvalidMetadata(t *testing.T) { require.NoError(t, err) require.Len(t, listed, 1) - _, err = m.ListInstancesForReconcile(context.Background()) + _, err = m.listInstancesForReconcile(context.Background()) require.Error(t, err) assert.ErrorContains(t, err, "load metadata for instance invalid") require.NoError(t, os.Remove(m.paths.InstanceMetadata("invalid"))) - listed, err = m.ListInstancesForReconcile(context.Background()) + listed, err = m.listInstancesForReconcile(context.Background()) require.NoError(t, err) require.Len(t, listed, 1) assert.Equal(t, "valid", listed[0].Id) @@ -70,7 +70,7 @@ func TestListInstancesForReconcileSkipsInstanceDeletedDuringListing(t *testing.T } done := make(chan result, 1) go func() { - listed, err := m.ListInstancesForReconcile(context.Background()) + listed, err := m.listInstancesForReconcile(context.Background()) done <- result{listed, err} }() time.Sleep(100 * time.Millisecond) diff --git a/lib/instances/snapshot.go b/lib/instances/snapshot.go index e30192ee9..1d8456ae5 100644 --- a/lib/instances/snapshot.go +++ b/lib/instances/snapshot.go @@ -266,6 +266,11 @@ func (m *manager) restoreSnapshot(ctx context.Context, id string, snapshotID str if sourceInst.State == StateRunning { return nil, fmt.Errorf("%w: cannot restore snapshot while source is %s", ErrInvalidState, sourceInst.State) } + if sourceMeta.GPURetainedForCleanup { + // Restoring would rebuild boot config from the snapshot record, whose + // retention flag is false, silently clearing the delete-only marker. + return nil, errVGPURetentionStub + } targetState, err := resolveSnapshotTargetState(rec.Snapshot.Kind, req.TargetState) if err != nil { diff --git a/lib/instances/snapshot_test.go b/lib/instances/snapshot_test.go index f8c022fe9..e4fcc048f 100644 --- a/lib/instances/snapshot_test.go +++ b/lib/instances/snapshot_test.go @@ -77,6 +77,43 @@ func TestCreateSnapshotRejectsVGPURetentionRecord(t *testing.T) { require.ErrorContains(t, err, "delete it to release the assignment") } +func TestRestoreSnapshotRejectsVGPURetentionRecord(t *testing.T) { + mgr, _ := setupTestManager(t) + ctx := context.Background() + + sourceID := "snapshot-vgpu-restore-retention" + createStoppedSnapshotSourceFixture(t, mgr, sourceID, sourceID, mgr.defaultHypervisor) + + snapshot, err := mgr.CreateSnapshot(ctx, sourceID, CreateSnapshotRequest{ + Kind: SnapshotKindStopped, + Name: "snapshot-vgpu-restore-retention", + }) + require.NoError(t, err) + + meta, err := mgr.loadMetadata(sourceID) + require.NoError(t, err) + meta.GPUProfile = "NVIDIA L40S-2Q" + meta.GPUFramework = devices.VGPUFramework("future-framework") + meta.GPUDevicePath = "/sys/bus/pci/devices/0000:82:00.4" + meta.GPURetainedForCleanup = true + require.NoError(t, mgr.saveMetadata(meta)) + + // Restoring into the delete-only stub would rebuild boot config from the + // snapshot record, whose retention flag is false, clearing the marker. + _, err = mgr.RestoreSnapshot(ctx, sourceID, snapshot.Id, RestoreSnapshotRequest{ + TargetState: StateStopped, + TargetHypervisor: mgr.defaultHypervisor, + }) + require.ErrorIs(t, err, ErrInvalidState) + require.ErrorContains(t, err, "delete it to release the assignment") + + // The retained assignment must survive the rejected restore for delete. + stored, err := mgr.loadMetadata(sourceID) + require.NoError(t, err) + assert.True(t, stored.GPURetainedForCleanup) + assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", stored.GPUDevicePath) +} + func TestRestoreSnapshotDoesNotResurrectStaleVGPUAssignment(t *testing.T) { mgr, _ := setupTestManager(t) ctx := context.Background() diff --git a/lib/instances/start.go b/lib/instances/start.go index a79663953..1089a35a3 100644 --- a/lib/instances/start.go +++ b/lib/instances/start.go @@ -185,6 +185,10 @@ func (m *manager) startInstance( wrapped := fmt.Errorf("create vGPU for profile %s: %w", stored.GPUProfile, err) if saveErr := m.saveMetadata(&retentionMeta); saveErr != nil { log.ErrorContext(ctx, "failed to retain vGPU assignment after create rollback failure", "instance_id", id, "error", saveErr) + // No on-disk record points at the device; retry the release + // in the background instead of waiting for the next startup + // reconcile. + m.scheduleOrphanedVGPURelease(ctx, retentionMeta.StoredMetadata) return nil, &VGPUCleanupPendingError{InstanceID: id, Err: fmt.Errorf("%w; retain assignment: %v", wrapped, saveErr)} } return nil, &VGPUCleanupPendingError{InstanceID: id, Retained: true, Err: wrapped} diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index 8ffe64dd8..b82a07d7d 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -17,7 +17,8 @@ const VGPUAssignmentStartupGracePeriod = 5 * time.Minute // VGPUCleanupPendingError reports a failed create whose vGPU release also // failed during rollback. When Retained is true, deleting the retained instance -// retries the release; otherwise startup reconciliation recovers the assignment. +// retries the release; otherwise a background retry and startup reconciliation +// recover the assignment. type VGPUCleanupPendingError struct { InstanceID string Retained bool @@ -28,7 +29,7 @@ func (e *VGPUCleanupPendingError) Error() string { if e.Retained { return fmt.Sprintf("%v; vGPU release failed during rollback, instance %s retains the assignment", e.Err, e.InstanceID) } - return fmt.Sprintf("%v; vGPU release failed during rollback and the retention record for instance %s could not be saved; the assignment is recovered on the next startup reconcile", e.Err, e.InstanceID) + return fmt.Sprintf("%v; vGPU release failed during rollback and the retention record for instance %s could not be saved; the release is retried in the background and by the next startup reconcile", e.Err, e.InstanceID) } func (e *VGPUCleanupPendingError) Unwrap() error { return e.Err } @@ -128,6 +129,9 @@ func (m *manager) cleanupStartVGPU(ctx context.Context, instanceID string, devic if meta, loadErr := m.loadMetadata(instanceID); loadErr == nil && storedVGPUDevicePath(&meta.StoredMetadata) == device.SysfsPath { return true, true } + // No on-disk record points at the device; retry the release in the + // background instead of waiting for the next startup reconcile. + m.scheduleOrphanedVGPURelease(ctx, cleanupMeta.StoredMetadata) return true, false } return retained, retained @@ -155,6 +159,15 @@ func restoreStartMutatedFields(dst, src *StoredMetadata) { } func (m *manager) releaseStoredVGPU(ctx context.Context, stored *StoredMetadata) error { + return m.releaseStoredVGPUExcluding(ctx, stored, stored.Id) +} + +// releaseStoredVGPUExcluding releases stored's assignment while treating +// excludeID's metadata as not a claimant. Callers releasing an instance's own +// persisted assignment exclude that instance; the orphan retry passes no +// exclusion because its instance may have been restarted onto the same VF, +// and that live claim must block the release. +func (m *manager) releaseStoredVGPUExcluding(ctx context.Context, stored *StoredMetadata, excludeID string) error { path := storedVGPUDevicePath(stored) if path != "" { // Vendor VFIO VFs are reused across instances, so the release must @@ -164,7 +177,7 @@ func (m *manager) releaseStoredVGPU(ctx context.Context, stored *StoredMetadata) claimed := false if stored.GPUFramework == devices.VGPUFrameworkVendorVFIO { var err error - claimed, err = m.vgpuAssignmentClaimedByLiveInstance(ctx, stored.Id, path) + claimed, err = m.vgpuAssignmentClaimedByLiveInstance(ctx, excludeID, path) if err != nil { return err } @@ -222,7 +235,7 @@ func (m *manager) vgpuAssignmentClaimedByLiveInstance(ctx context.Context, exclu return false, fmt.Errorf("cannot confirm liveness of vGPU claimant %s on %s: %w", id, devicePath, err) } } - live, remaining := vgpuAssignmentLiveness(stored, time.Now(), pid > 0) + live, remaining := vgpuAssignmentLiveness(stored, m.nowUTC(), pid > 0) if pid > 0 && live { return true, nil } diff --git a/lib/instances/vgpu_orphan.go b/lib/instances/vgpu_orphan.go index d85a68bf3..513cf9bc9 100644 --- a/lib/instances/vgpu_orphan.go +++ b/lib/instances/vgpu_orphan.go @@ -15,12 +15,13 @@ const ( defaultOrphanedVGPUReleaseRetryDelay = 30 * time.Second ) -// scheduleOrphanedVGPURelease retries a vGPU release that failed during a -// completed delete, off the request path: a GPU-busy VMM routinely outlives -// delete's force-kill wait, and once metadata is deleted nothing else -// releases the VF until startup reconciliation. Each attempt re-runs -// releaseStoredVGPU, so the claim scan and destroy guards apply on every -// retry. The queue is in-memory only; a restart abandons it and startup +// scheduleOrphanedVGPURelease retries a vGPU release for an assignment no +// on-disk metadata points at anymore: a release that failed during a +// completed delete (a GPU-busy VMM routinely outlives delete's force-kill +// wait), or a rollback whose retention record could not be saved. Without a +// record, nothing else releases the VF until startup reconciliation. Each +// attempt re-runs the release, so the claim scan and destroy guards apply on +// every retry. The queue is in-memory only; a restart abandons it and startup // reconciliation sweeps the VF. func (m *manager) scheduleOrphanedVGPURelease(ctx context.Context, stored StoredMetadata) { path := storedVGPUDevicePath(&stored) @@ -56,7 +57,9 @@ func (m *manager) retryOrphanedVGPURelease(ctx context.Context, stored StoredMet }() for attempt := 1; attempt <= orphanedVGPUReleaseMaxAttempts; attempt++ { time.Sleep(delay) - if err := m.releaseStoredVGPU(ctx, &stored); err != nil { + // No claim-scan exclusion: unlike delete, a failed start keeps its + // instance record, and a restarted instance may hold this same VF. + if err := m.releaseStoredVGPUExcluding(ctx, &stored, ""); err != nil { log.WarnContext(ctx, "orphaned vGPU release retry failed", "instance_id", stored.Id, "device_path", path, "attempt", attempt, "error", err) continue diff --git a/lib/instances/vgpu_reconcile.go b/lib/instances/vgpu_reconcile.go index 85d3a6e0d..4890b216a 100644 --- a/lib/instances/vgpu_reconcile.go +++ b/lib/instances/vgpu_reconcile.go @@ -8,8 +8,13 @@ import ( "github.com/kernel/hypeman/lib/logger" ) +// vgpuReconcileListRetryDelay spaces retries of the vendor VFIO sweep when +// the instance listing fails: without a retry, one transient stat error at +// startup would disable orphan recovery until the next restart. +const vgpuReconcileListRetryDelay = time.Minute + func (m *manager) liveVGPUReconcileProtection(ctx context.Context) (map[string]struct{}, time.Duration, error) { - allInstances, err := m.ListInstancesForReconcile(ctx) + allInstances, err := m.listInstancesForReconcile(ctx) if err != nil { return nil, 0, err } @@ -20,7 +25,7 @@ func (m *manager) liveVGPUReconcileProtection(ctx context.Context) (map[string]s if stored.GPUDevicePath == "" { continue } - livePID := stored.HypervisorPID != nil && HypervisorMayBeAlive(stored.HypervisorProcessIdentity, stored.SocketPath) + livePID := stored.HypervisorPID != nil && hypervisorMayBeAlive(stored.HypervisorProcessIdentity, stored.SocketPath) live, remaining := vgpuAssignmentLiveness(stored, m.nowUTC(), livePID) if !live { continue @@ -41,7 +46,10 @@ func (m *manager) ReconcileVGPUs(ctx context.Context) { if err != nil { log.ErrorContext(ctx, "failed to list instances for vGPU reconcile protection; skipping vendor VFIO reconcile, mdev reconcile still runs", "error", err) protected = make(map[string]struct{}) - retryAfter = 0 + retryAfter = vgpuReconcileListRetryDelay + if m.vgpuReconcileRetryDelay > 0 { + retryAfter = m.vgpuReconcileRetryDelay + } } if err := devices.ReconcileVGPUs(ctx, protected, sweepVendorVFIO); err != nil { log.WarnContext(ctx, "failed to reconcile vGPU devices", "error", err) @@ -49,12 +57,19 @@ func (m *manager) ReconcileVGPUs(ctx context.Context) { if retryAfter <= 0 { return } + // One pending retry at a time: overlapping calls would fork parallel + // retry chains. + if !m.vgpuReconcileRetryPending.CompareAndSwap(false, true) { + return + } go func() { timer := time.NewTimer(retryAfter) defer timer.Stop() select { case <-ctx.Done(): + m.vgpuReconcileRetryPending.Store(false) case <-timer.C: + m.vgpuReconcileRetryPending.Store(false) m.ReconcileVGPUs(ctx) } }() diff --git a/lib/instances/vgpu_reconcile_test.go b/lib/instances/vgpu_reconcile_test.go index 1fbd90384..6fd299af7 100644 --- a/lib/instances/vgpu_reconcile_test.go +++ b/lib/instances/vgpu_reconcile_test.go @@ -1,7 +1,10 @@ package instances import ( + "context" + "os" "os/exec" + "path/filepath" "testing" "time" @@ -41,6 +44,32 @@ func TestLiveVGPUReconcileProtectionBoundsStartupProtection(t *testing.T) { assert.Contains(t, protected, "/sys/bus/pci/devices/0000:82:00.8") } +func TestReconcileVGPUsRetriesAfterListingFailure(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("root bypasses directory permissions") + } + + m := &manager{paths: paths.New(t.TempDir()), vgpuReconcileRetryDelay: 250 * time.Millisecond} + const id = "unreadable" + require.NoError(t, m.ensureDirectories(id)) + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{Id: id}})) + instanceDir := filepath.Dir(m.paths.InstanceMetadata(id)) + require.NoError(t, os.Chmod(instanceDir, 0o000)) + t.Cleanup(func() { _ = os.Chmod(instanceDir, 0o755) }) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + m.ReconcileVGPUs(ctx) + require.True(t, m.vgpuReconcileRetryPending.Load(), + "a listing failure must schedule a retry instead of disabling the vendor sweep until restart") + + // Once the listing recovers, the retry runs the sweep and stops rearming. + require.NoError(t, os.Chmod(instanceDir, 0o755)) + require.Eventually(t, func() bool { + return !m.vgpuReconcileRetryPending.Load() + }, 5*time.Second, 10*time.Millisecond) +} + func TestVGPUAssignmentLiveness(t *testing.T) { now := time.Now().UTC() recent := now.Add(-time.Minute) diff --git a/lib/instances/vgpu_retention.go b/lib/instances/vgpu_retention.go index a1a1aae0d..f6cd4b02b 100644 --- a/lib/instances/vgpu_retention.go +++ b/lib/instances/vgpu_retention.go @@ -30,11 +30,6 @@ func (r *vgpuRetention) retainFromDevice(stub StoredMetadata, device *devices.VG r.retained = true } -func (r *vgpuRetention) retain(stub *StoredMetadata) { - r.stub = stub - r.retained = stub != nil -} - func (r *vgpuRetention) markRetained(persisted bool) { r.retained = true r.persisted = persisted @@ -70,6 +65,9 @@ func (m *manager) persistVGPURetention(ctx context.Context, retention *vgpuReten if err := m.deleteInstanceData(id); err != nil { log.ErrorContext(ctx, "failed to delete stale instance data after retention failure", "instance_id", id, "error", err) } + // No on-disk record points at the device; retry the release in the + // background instead of waiting for the next startup reconcile. + m.scheduleOrphanedVGPURelease(ctx, *retainedVGPU) return false } if err := m.ensureDirectories(id); err != nil { diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index c5ef4911b..ea3b18429 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -19,8 +19,7 @@ import ( ) func persistTestVGPURetention(m *manager, ctx context.Context, id string, stub *StoredMetadata) bool { - retention := vgpuRetention{instanceID: id} - retention.retain(stub) + retention := vgpuRetention{instanceID: id, stub: stub, retained: stub != nil} m.persistVGPURetention(ctx, &retention) return retention.persisted } @@ -101,6 +100,11 @@ func TestCleanupFailedCreateReportsUnpersistedRetention(t *testing.T) { assert.False(t, persistTestVGPURetention(m, context.Background(), stored.Id, stored)) _, err := m.loadMetadata(id) require.Error(t, err) + + m.orphanedVGPUMu.Lock() + _, queued := m.orphanedVGPUs[stored.GPUDevicePath] + m.orphanedVGPUMu.Unlock() + assert.True(t, queued, "unpersisted retention must queue a background release") } func TestCleanupFailedCreateReportsRetainedWhenFullMetadataSurvives(t *testing.T) { @@ -157,7 +161,7 @@ func TestVGPUCleanupPendingErrorUnwraps(t *testing.T) { unpersisted := &VGPUCleanupPendingError{InstanceID: "inst-1", Err: cause} assert.ErrorIs(t, unpersisted, cause) - assert.Equal(t, "boot failed; vGPU release failed during rollback and the retention record for instance inst-1 could not be saved; the assignment is recovered on the next startup reconcile", unpersisted.Error()) + assert.Equal(t, "boot failed; vGPU release failed during rollback and the retention record for instance inst-1 could not be saved; the release is retried in the background and by the next startup reconcile", unpersisted.Error()) } func TestVGPUDevicePendingCleanup(t *testing.T) { @@ -334,6 +338,11 @@ func TestStartReportsUnretainedVGPUWhenRetentionSaveFails(t *testing.T) { stored, err := m.loadMetadata(id) require.NoError(t, err) assert.Empty(t, stored.GPUDevicePath, "retention save failed, so no assignment should be recorded") + + m.orphanedVGPUMu.Lock() + _, queued := m.orphanedVGPUs[device.SysfsPath] + m.orphanedVGPUMu.Unlock() + assert.True(t, queued, "unpersisted retention must queue a background release") } func TestStartDoesNotRestrictVGPUHypervisor(t *testing.T) { diff --git a/lib/instances/wait_test.go b/lib/instances/wait_test.go index 3ab02e402..dbb630185 100644 --- a/lib/instances/wait_test.go +++ b/lib/instances/wait_test.go @@ -32,7 +32,6 @@ func (s *stubManager) GetInstance(ctx context.Context, id string) (*Instance, er func (s *stubManager) ListInstances(context.Context, *ListInstancesFilter) ([]Instance, error) { return nil, nil } -func (s *stubManager) ReconcileVGPUs(context.Context) {} func (s *stubManager) ListSnapshots(context.Context, *ListSnapshotsFilter) ([]Snapshot, error) { return nil, nil } From e361540757632342d27f238135fd93c93d9c9d98 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Mon, 24 Aug 2026 20:08:05 +0000 Subject: [PATCH 60/76] Validate live vGPU claim test identity --- lib/instances/vgpu_test.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index ea3b18429..7ae6f3cdf 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -526,10 +526,14 @@ func TestVGPUAssignmentClaimedByLiveInstanceNormalizesLegacyMdevPath(t *testing. require.NoError(t, m.ensureDirectories("legacy-claimant")) pid := os.Getpid() require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ - Id: "legacy-claimant", - Name: "legacy-claimant", - GPUMdevUUID: "legacy-uuid", - HypervisorProcessIdentity: HypervisorProcessIdentity{HypervisorPID: &pid}, + Id: "legacy-claimant", + Name: "legacy-claimant", + GPUMdevUUID: "legacy-uuid", + HypervisorProcessIdentity: HypervisorProcessIdentity{ + HypervisorPID: &pid, + HypervisorStartTime: processStartTime(pid), + HypervisorBootID: hostBootID(), + }, }})) claimed, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "other-instance", "/sys/bus/mdev/devices/legacy-uuid") From a9387fc7d2257b28dd2cf89f610501a4c3255117 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Mon, 24 Aug 2026 21:00:23 +0000 Subject: [PATCH 61/76] Clean guest data before retaining vGPU assignment --- lib/instances/vgpu_retention.go | 5 +++++ lib/instances/vgpu_test.go | 17 ++++++++++++++--- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/lib/instances/vgpu_retention.go b/lib/instances/vgpu_retention.go index f6cd4b02b..eaca96289 100644 --- a/lib/instances/vgpu_retention.go +++ b/lib/instances/vgpu_retention.go @@ -70,6 +70,11 @@ func (m *manager) persistVGPURetention(ctx context.Context, retention *vgpuReten m.scheduleOrphanedVGPURelease(ctx, *retainedVGPU) return false } + if err := m.deleteInstanceData(id); err != nil { + log.ErrorContext(ctx, "failed to clean instance data before retaining vGPU assignment", "instance_id", id, "error", err) + retention.persisted = retentionSurvives() + return + } if err := m.ensureDirectories(id); err != nil { log.ErrorContext(ctx, "failed to retain instance data after vGPU cleanup failure", "instance_id", id, "error", err) retention.persisted = retentionSurvives() diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index 7ae6f3cdf..cb54c509b 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -49,8 +49,16 @@ func TestCleanupFailedCreateRetainsVGPUAssignment(t *testing.T) { HypervisorType: "qemu", DataDir: m.paths.InstanceDir("failed-create"), } + require.NoError(t, m.ensureDirectories(stored.Id)) + require.NoError(t, os.WriteFile(m.paths.InstanceOverlay(stored.Id), []byte("overlay"), 0o644)) + require.NoError(t, os.WriteFile(m.paths.InstanceConfigDisk(stored.Id), []byte("config"), 0o644)) + require.NoError(t, os.MkdirAll(m.paths.InstanceVolumeOverlaysDir(stored.Id), 0o755)) + require.NoError(t, os.WriteFile(m.paths.InstanceVolumeOverlay(stored.Id, "volume"), []byte("volume overlay"), 0o644)) assert.True(t, persistTestVGPURetention(m, context.Background(), stored.Id, stored)) + assert.NoFileExists(t, m.paths.InstanceOverlay(stored.Id)) + assert.NoFileExists(t, m.paths.InstanceConfigDisk(stored.Id)) + assert.NoDirExists(t, m.paths.InstanceVolumeOverlaysDir(stored.Id)) retained, err := m.loadMetadata(stored.Id) require.NoError(t, err) @@ -85,12 +93,15 @@ func TestCleanupFailedCreateDeletesDataWithoutRetainedVGPU(t *testing.T) { } func TestCleanupFailedCreateReportsUnpersistedRetention(t *testing.T) { - t.Parallel() + if os.Geteuid() == 0 { + t.Skip("root bypasses directory permissions") + } m := &manager{paths: paths.New(t.TempDir())} const id = "failed-create" - require.NoError(t, m.ensureDirectories(id)) - require.NoError(t, os.Mkdir(m.paths.InstanceMetadata(id), 0o755)) + require.NoError(t, os.MkdirAll(m.paths.GuestsDir(), 0o755)) + require.NoError(t, os.Chmod(m.paths.GuestsDir(), 0o555)) + t.Cleanup(func() { _ = os.Chmod(m.paths.GuestsDir(), 0o755) }) stored := &StoredMetadata{ Id: id, From 9b3ece3d35a1cea4cfb0006f6ca67241ae2b1fe7 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Tue, 25 Aug 2026 14:21:58 +0000 Subject: [PATCH 62/76] Retry failed vGPU reconciliation --- lib/instances/manager.go | 7 ++++--- lib/instances/vgpu_reconcile.go | 22 ++++++++++++++++------ lib/instances/vgpu_reconcile_test.go | 24 ++++++++++++++++++++++++ 3 files changed, 44 insertions(+), 9 deletions(-) diff --git a/lib/instances/manager.go b/lib/instances/manager.go index da1fe59d3..91b25aebc 100644 --- a/lib/instances/manager.go +++ b/lib/instances/manager.go @@ -186,6 +186,7 @@ type manager struct { deleteInstanceFn func(context.Context, string) error createVGPU func(context.Context, string, string) (*devices.VGPUDevice, error) destroyVGPU func(context.Context, devices.VGPUAssignment) error + reconcileVGPUDevices func(context.Context, map[string]struct{}, bool) error deleteSnapshotFn func(context.Context, string) error ttlReaperDeleteTimeout time.Duration egressProxy *egressproxy.Service @@ -222,9 +223,9 @@ type manager struct { orphanedVGPUs map[string]struct{} orphanedVGPURetryDelay time.Duration - // One pending vGPU reconcile retry at a time, for both the startup-grace - // and listing-failure paths. vgpuReconcileRetryDelay overrides the - // listing-failure delay in tests; zero means the default. + // One pending vGPU reconcile retry at a time, for startup grace and + // reconciliation failures. vgpuReconcileRetryDelay overrides the retry + // delay in tests; zero means the default. vgpuReconcileRetryPending atomic.Bool vgpuReconcileRetryDelay time.Duration diff --git a/lib/instances/vgpu_reconcile.go b/lib/instances/vgpu_reconcile.go index 4890b216a..24be04fe3 100644 --- a/lib/instances/vgpu_reconcile.go +++ b/lib/instances/vgpu_reconcile.go @@ -8,10 +8,9 @@ import ( "github.com/kernel/hypeman/lib/logger" ) -// vgpuReconcileListRetryDelay spaces retries of the vendor VFIO sweep when -// the instance listing fails: without a retry, one transient stat error at -// startup would disable orphan recovery until the next restart. -const vgpuReconcileListRetryDelay = time.Minute +// vgpuReconcileFailureRetryDelay spaces retries after a transient metadata or +// device error would otherwise disable orphan recovery until the next restart. +const vgpuReconcileFailureRetryDelay = time.Minute func (m *manager) liveVGPUReconcileProtection(ctx context.Context) (map[string]struct{}, time.Duration, error) { allInstances, err := m.listInstancesForReconcile(ctx) @@ -46,13 +45,24 @@ func (m *manager) ReconcileVGPUs(ctx context.Context) { if err != nil { log.ErrorContext(ctx, "failed to list instances for vGPU reconcile protection; skipping vendor VFIO reconcile, mdev reconcile still runs", "error", err) protected = make(map[string]struct{}) - retryAfter = vgpuReconcileListRetryDelay + retryAfter = vgpuReconcileFailureRetryDelay if m.vgpuReconcileRetryDelay > 0 { retryAfter = m.vgpuReconcileRetryDelay } } - if err := devices.ReconcileVGPUs(ctx, protected, sweepVendorVFIO); err != nil { + reconcileDevices := m.reconcileVGPUDevices + if reconcileDevices == nil { + reconcileDevices = devices.ReconcileVGPUs + } + if err := reconcileDevices(ctx, protected, sweepVendorVFIO); err != nil { log.WarnContext(ctx, "failed to reconcile vGPU devices", "error", err) + deviceRetryAfter := vgpuReconcileFailureRetryDelay + if m.vgpuReconcileRetryDelay > 0 { + deviceRetryAfter = m.vgpuReconcileRetryDelay + } + if retryAfter <= 0 || deviceRetryAfter < retryAfter { + retryAfter = deviceRetryAfter + } } if retryAfter <= 0 { return diff --git a/lib/instances/vgpu_reconcile_test.go b/lib/instances/vgpu_reconcile_test.go index 6fd299af7..815415d09 100644 --- a/lib/instances/vgpu_reconcile_test.go +++ b/lib/instances/vgpu_reconcile_test.go @@ -2,9 +2,11 @@ package instances import ( "context" + "errors" "os" "os/exec" "path/filepath" + "sync/atomic" "testing" "time" @@ -70,6 +72,28 @@ func TestReconcileVGPUsRetriesAfterListingFailure(t *testing.T) { }, 5*time.Second, 10*time.Millisecond) } +func TestReconcileVGPUsRetriesAfterDeviceFailure(t *testing.T) { + var calls atomic.Int32 + m := &manager{ + paths: paths.New(t.TempDir()), + vgpuReconcileRetryDelay: 10 * time.Millisecond, + reconcileVGPUDevices: func(context.Context, map[string]struct{}, bool) error { + if calls.Add(1) == 1 { + return errors.New("transient device error") + } + return nil + }, + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + m.ReconcileVGPUs(ctx) + require.True(t, m.vgpuReconcileRetryPending.Load()) + require.Eventually(t, func() bool { + return calls.Load() >= 2 && !m.vgpuReconcileRetryPending.Load() + }, 5*time.Second, 10*time.Millisecond) +} + func TestVGPUAssignmentLiveness(t *testing.T) { now := time.Now().UTC() recent := now.Add(-time.Minute) From c2737529ec245aa6e995f98e27e3e8ddfceb2505 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Tue, 25 Aug 2026 15:19:57 +0000 Subject: [PATCH 63/76] Trim vGPU lifecycle comments and tests --- cmd/api/api/instances.go | 7 ---- cmd/api/api/instances_test.go | 5 --- cmd/api/main.go | 4 --- lib/hypervisor/qemu/process.go | 9 ++--- lib/instances/create.go | 4 --- lib/instances/delete.go | 9 +---- lib/instances/fork_test.go | 2 -- lib/instances/lifecycle_noop_test.go | 7 ---- lib/instances/manager.go | 14 -------- lib/instances/metrics.go | 3 -- lib/instances/process_identity.go | 15 ++------ lib/instances/process_identity_linux_test.go | 27 +-------------- lib/instances/query_test.go | 8 ----- lib/instances/snapshot.go | 2 -- lib/instances/snapshot_test.go | 5 --- lib/instances/start.go | 4 --- lib/instances/storage.go | 5 --- lib/instances/types.go | 15 ++++---- lib/instances/vgpu.go | 36 ++------------------ lib/instances/vgpu_orphan.go | 15 -------- lib/instances/vgpu_orphan_test.go | 16 --------- lib/instances/vgpu_reconcile.go | 4 --- lib/instances/vgpu_reconcile_test.go | 30 ---------------- lib/instances/vgpu_retention.go | 2 -- lib/instances/vgpu_test.go | 23 ------------- 25 files changed, 14 insertions(+), 257 deletions(-) diff --git a/cmd/api/api/instances.go b/cmd/api/api/instances.go index 46be5cec4..e424da8e4 100644 --- a/cmd/api/api/instances.go +++ b/cmd/api/api/instances.go @@ -364,8 +364,6 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst if err != nil { var vgpuPending *instances.VGPUCleanupPendingError switch { - // Checked first: it wraps the original create error, so a later - // errors.Is case would match the cause and hide the pending vGPU cleanup. case errors.As(err, &vgpuPending): log.ErrorContext(ctx, "failed to create instance", "error", err, "image", request.Body.Image) message, inner := vgpuCleanupPendingDetail(vgpuPending, "create", "delete it to retry") @@ -435,9 +433,6 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst return oapi.CreateInstance201JSONResponse(instanceToOAPI(*inst)), nil } -// vgpuCleanupPendingDetail renders a pending vGPU cleanup into the message -// and inner error detail shared by the create and start handlers. The -// retained guidance names the verb-specific way to release the assignment. func vgpuCleanupPendingDetail(pending *instances.VGPUCleanupPendingError, action, retainedGuidance string) (string, *oapi.ErrorDetail) { message := fmt.Sprintf("failed to %s instance: %v", action, pending) innerCode := "vgpu_unretained_instance" @@ -863,8 +858,6 @@ func (s *ApiService) StartInstance(ctx context.Context, request oapi.StartInstan if err != nil { var vgpuPending *instances.VGPUCleanupPendingError switch { - // Checked first: it wraps the original start error, so a later - // errors.Is case would match the cause and hide the pending vGPU cleanup. case errors.As(err, &vgpuPending): log.ErrorContext(ctx, "failed to start instance", "error", err) message, inner := vgpuCleanupPendingDetail(vgpuPending, "start", "delete it or retry start to release it") diff --git a/cmd/api/api/instances_test.go b/cmd/api/api/instances_test.go index 738cf547c..fd506023d 100644 --- a/cmd/api/api/instances_test.go +++ b/cmd/api/api/instances_test.go @@ -57,8 +57,6 @@ func (m createErrorInstanceManager) CreateInstance(context.Context, instances.Cr return nil, m.err } -// A retained-assignment error must win over the mapping of the create error -// it wraps, or the response omits the instance the caller has to delete. func TestCreateInstance_VGPUCleanupPendingBeatsWrappedErrorMapping(t *testing.T) { t.Parallel() svc := newTestService(t) @@ -1082,9 +1080,6 @@ func TestRestoreInstance_ErrorMapping(t *testing.T) { } } -// A retained-assignment error must win over the mapping of the start error -// it wraps, or the response omits the pending vGPU cleanup the caller has to -// resolve. func TestStartInstance_VGPUCleanupPendingBeatsWrappedErrorMapping(t *testing.T) { t.Parallel() diff --git a/cmd/api/main.go b/cmd/api/main.go index 20ac8dc60..3bd0fff39 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -384,10 +384,6 @@ func run() error { return fmt.Errorf("reconcile device state: %w", err) } - // Reconcile vGPU devices (clears orphaned vGPUs from previous runs). - // Type-asserted rather than added to instances.Manager so alternate - // Manager implementations compiled against the public module keep - // building without this startup-only method. logger.Info("Reconciling vGPU devices...") if r, ok := app.InstanceManager.(interface{ ReconcileVGPUs(context.Context) }); ok { r.ReconcileVGPUs(ctx) diff --git a/lib/hypervisor/qemu/process.go b/lib/hypervisor/qemu/process.go index de560c10d..f16d606eb 100644 --- a/lib/hypervisor/qemu/process.go +++ b/lib/hypervisor/qemu/process.go @@ -225,11 +225,8 @@ func buildQMPArgs(socketPath string) []string { } type startedProcess struct { - pid int - socketPath string - // termGrace, when non-zero, makes cleanup send SIGTERM and wait this long - // before SIGKILL. Set for VFIO-attached processes: SIGKILL during vGPU - // plugin init can wedge the VF until its parent GPU is SR-IOV cycled. + pid int + socketPath string termGrace time.Duration waitDone chan error waitConsumed bool @@ -281,7 +278,6 @@ func (p *startedProcess) wait() error { return err } -// waitFor waits up to d for the process to exit, returning whether it did. func (p *startedProcess) waitFor(d time.Duration) bool { if _, exited := p.checkExited(); exited { return true @@ -392,7 +388,6 @@ func (s *Starter) startQEMUProcess(ctx context.Context, p *paths.Paths, version } pid := proc.pid - // Only failed starts pay the VFIO termination grace. proc.termGrace = termGrace log.DebugContext(processCtx, "QEMU process started", "pid", pid, "duration_ms", time.Since(processStartTime).Milliseconds()) diff --git a/lib/instances/create.go b/lib/instances/create.go index de301a95d..525f2c262 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -280,7 +280,6 @@ func (m *manager) createInstance( var gpuAssignedAt *time.Time retention := vgpuRetention{instanceID: id} - // Setup cleanup stack early so device attachment errors trigger cleanup. defer retention.deferWrapPending(&retErr) cu := cleanup.Make(func() { log.DebugContext(ctx, "cleaning up instance on error", "instance_id", id) @@ -300,9 +299,6 @@ func (m *manager) createInstance( // Handle vGPU profile request if req.GPU != nil && req.GPU.Profile != "" { - // Identity fields a retention record keeps when rollback cannot - // release the assignment, so it lists as a recognizable, deletable - // instance. Create has already failed on a nil starter by this point. retentionStub := func() StoredMetadata { return StoredMetadata{ Id: id, diff --git a/lib/instances/delete.go b/lib/instances/delete.go index 2f90a8552..160ae807d 100644 --- a/lib/instances/delete.go +++ b/lib/instances/delete.go @@ -140,19 +140,12 @@ func (m *manager) deleteInstanceWithOptions( } m.closeFirecrackerUFFDSession(ctx, stored) - // 5b. Release the vGPU assignment if present, before any network, device, - // or volume teardown. Release failure is logged and the delete continues, - // matching the pre-refactor contract: the VMM is already confirmed dead, - // the guards inside the release never destroy a device they cannot prove - // is unowned, and a skipped release is recovered by the background retry - // below or, after a restart, by startup reconciliation. + // Release before deleting metadata so a failed release can be retried safely. hadVGPUAssignment := storedVGPUDevicePath(stored) != "" if hadVGPUAssignment { log.InfoContext(ctx, "destroying vGPU", "instance_id", id, "uuid", stored.GPUMdevUUID) } if err := m.releaseStoredVGPU(ctx, stored); err != nil { - // Log error but continue with cleanup; the background retry releases - // the VF once the metadata is gone. log.WarnContext(ctx, "failed to destroy vGPU, continuing with cleanup", "instance_id", id, "uuid", stored.GPUMdevUUID, "error", err) m.scheduleOrphanedVGPURelease(ctx, *stored) } else if hadVGPUAssignment { diff --git a/lib/instances/fork_test.go b/lib/instances/fork_test.go index ef802d1ad..c0bed670e 100644 --- a/lib/instances/fork_test.go +++ b/lib/instances/fork_test.go @@ -81,8 +81,6 @@ func TestForkInstanceRejectsVGPURetentionRecord(t *testing.T) { meta.GPURetainedForCleanup = true require.NoError(t, manager.saveMetadata(meta)) - // The delete-only retention stub has no boot configuration, so a fork of - // it could never boot; only delete may act on it. _, err = manager.ForkInstance(ctx, sourceID, ForkInstanceRequest{Name: "fork-vgpu-retention-copy"}) require.ErrorIs(t, err, ErrInvalidState) require.ErrorContains(t, err, "delete it to release the assignment") diff --git a/lib/instances/lifecycle_noop_test.go b/lib/instances/lifecycle_noop_test.go index a43efabb0..90f2e7aa1 100644 --- a/lib/instances/lifecycle_noop_test.go +++ b/lib/instances/lifecycle_noop_test.go @@ -198,10 +198,6 @@ func TestDeletePersistsVGPUReleaseBeforeTeardown(t *testing.T) { assert.Equal(t, restartpolicy.BlockedReasonManualStop, persisted.RestartStatus.BlockedReason) } -// A failed create whose vGPU release also failed retains a minimal -// GPU-fields-only stub, and the API tells the caller to delete it to retry -// the release. Exercise that recovery path against the exact stub shape -// persistVGPURetention writes. func TestDeleteReleasesRetainedCreateStub(t *testing.T) { p := paths.New(t.TempDir()) var destroyed []devices.VGPUAssignment @@ -342,7 +338,6 @@ func TestStartRejectsVGPURetentionRecord(t *testing.T) { require.ErrorIs(t, err, ErrInvalidState) require.ErrorContains(t, err, "delete it to release the assignment") - // The retained assignment must survive the rejected start for delete. stored, err := m.loadMetadata(id) require.NoError(t, err) assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", stored.GPUDevicePath) @@ -381,8 +376,6 @@ func TestStopStoppedInstanceLeavesRetentionStubForDelete(t *testing.T) { require.NoError(t, err) require.NotNil(t, inst) - // Retention stubs are delete-only: releasing on stop would leave a stub - // whose start/fork/snapshot errors still claim a retained assignment. stored, err := m.loadMetadata(id) require.NoError(t, err) assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", stored.GPUDevicePath) diff --git a/lib/instances/manager.go b/lib/instances/manager.go index 91b25aebc..7cd94262a 100644 --- a/lib/instances/manager.go +++ b/lib/instances/manager.go @@ -215,22 +215,13 @@ type manager struct { // Periodic TAP garbage collection reconciler. tapGCOnce sync.Once - // vGPU assignments that survived a completed delete, keyed by device - // path, each with a background release retry in flight. - // orphanedVGPURetryDelay overrides the retry delay in tests; zero means - // the default. orphanedVGPUMu sync.Mutex orphanedVGPUs map[string]struct{} orphanedVGPURetryDelay time.Duration - // One pending vGPU reconcile retry at a time, for startup grace and - // reconciliation failures. vgpuReconcileRetryDelay overrides the retry - // delay in tests; zero means the default. vgpuReconcileRetryPending atomic.Bool vgpuReconcileRetryDelay time.Duration - // vgpuInitTermGrace overrides terminateThenKill's SIGTERM wait for vGPU - // instances still initializing; zero means the default. vgpuInitTermGrace time.Duration // Hypervisor support @@ -758,9 +749,6 @@ func (m *manager) DefaultHypervisor() hypervisor.Type { return m.defaultHypervisor } -// listInstancesForReconcile returns every instance's stored metadata or an -// invalid metadata error. It does not derive state: hydration would query -// every hypervisor on the host before the API serves. func (m *manager) listInstancesForReconcile(ctx context.Context) ([]Instance, error) { files, err := m.listMetadataFilesStrict() if err != nil { @@ -772,8 +760,6 @@ func (m *manager) listInstancesForReconcile(ctx context.Context) ([]Instance, er meta, err := m.loadMetadata(id) if err != nil { if errors.Is(err, ErrNotFound) { - // Deleted between listing and load; failing instead would - // disable the vendor VFIO sweep whenever it races a delete. continue } return nil, fmt.Errorf("load metadata for instance %s: %w", id, err) diff --git a/lib/instances/metrics.go b/lib/instances/metrics.go index 085216fdf..de71c2686 100644 --- a/lib/instances/metrics.go +++ b/lib/instances/metrics.go @@ -573,9 +573,6 @@ func (m *manager) recordTimeToRunning(ctx context.Context, stored *StoredMetadat m.metrics.timeToRunning.Record(ctx, duration, metric.WithAttributes(attrs...)) } -// recordVGPUOrphanReleaseAbandoned records an orphaned vGPU release retry -// loop giving up: the VF stays allocated until startup reconciliation or -// manual remediation, so it must be visible beyond a log line. func (m *manager) recordVGPUOrphanReleaseAbandoned(ctx context.Context) { if m.metrics == nil { return diff --git a/lib/instances/process_identity.go b/lib/instances/process_identity.go index cd8ff2fe1..02f43fcda 100644 --- a/lib/instances/process_identity.go +++ b/lib/instances/process_identity.go @@ -27,8 +27,6 @@ const linuxBootIDPath = "/proc/sys/kernel/random/boot_id" // does not unstick it, so the wait is short to keep stop and delete fast. const hypervisorSIGKILLWaitTimeout = 2 * time.Second -// vgpuTermGrace returns the SIGTERM wait used before hard-killing a vGPU -// hypervisor. func (m *manager) vgpuTermGrace() time.Duration { if m.vgpuInitTermGrace > 0 { return m.vgpuInitTermGrace @@ -36,13 +34,7 @@ func (m *manager) vgpuTermGrace() time.Duration { return hypervisor.VFIOTermGrace } -// terminateThenKill hard-kills the hypervisor process, first giving any -// instance with VFIO devices (a vGPU VF or passthrough PCI devices, matching -// the QEMU-side vfioTermGraceFor) a SIGTERM grace: SIGKILL during guest -// driver init can wedge the device until its parent GPU is SR-IOV cycled -// (see lib/devices/GPU.md). The grace applies in every state because the -// instance reports Running seconds before driver init finishes and nothing -// host-side observes that boundary. +// SIGKILL during guest driver init can wedge a VF until the parent GPU is reset. func (m *manager) terminateThenKill(ctx context.Context, inst *Instance, pid int) error { if inst.GPUProfile != "" || len(inst.Devices) > 0 { if syscall.Kill(pid, syscall.SIGTERM) == nil && WaitForProcessExit(pid, m.vgpuTermGrace()) { @@ -209,10 +201,7 @@ func classifyResolvedHypervisorOwner(socketPath string, stored, resolved int, er return 0, fmt.Errorf("cannot confirm ownership of socket %s: %w", socketPath, err) } -// hypervisorMayBeAlive reports whether the recorded hypervisor process may -// still be running. It fails open (unresolvable ownership returns true, the -// safe direction for reconcile protection and claim checks); do not use it -// to authorize teardown. +// Ambiguous ownership is treated as live; this must not authorize teardown. func hypervisorMayBeAlive(id HypervisorProcessIdentity, socketPath string) bool { pid, err := resolveLiveHypervisorPID(id, socketPath) return err != nil || pid > 0 diff --git a/lib/instances/process_identity_linux_test.go b/lib/instances/process_identity_linux_test.go index e090b9a98..b208a6ef3 100644 --- a/lib/instances/process_identity_linux_test.go +++ b/lib/instances/process_identity_linux_test.go @@ -692,9 +692,6 @@ func TestResolveRuntimeHypervisorPIDMintsIdentityOnlyWhenConfirmed(t *testing.T) }) } -// startTrapProcess starts a shell with the given TERM trap action (empty -// ignores the signal) and blocks until the trap is installed. It returns the -// PID and its boot-scoped identity. func startTrapProcess(t *testing.T, trapAction string) (int, HypervisorProcessIdentity) { t.Helper() script := fmt.Sprintf("trap '%s' TERM; echo ready; sleep 30 & wait", trapAction) @@ -717,7 +714,7 @@ func startTrapProcess(t *testing.T, trapAction string) (int, HypervisorProcessId return pid, HypervisorProcessIdentity{HypervisorPID: &pid, HypervisorStartTime: startTime, HypervisorBootID: hostBootID()} } -func TestKillHypervisorSIGTERMsInitializingVGPUHypervisor(t *testing.T) { +func TestKillHypervisorSIGTERMsVGPUHypervisor(t *testing.T) { markerPath := filepath.Join(t.TempDir(), "terminated") pid, identity := startTrapProcess(t, "touch "+markerPath+"; exit 0") socketPath := filepath.Join(t.TempDir(), "missing.sock") @@ -757,28 +754,6 @@ func TestKillHypervisorEscalatesToSIGKILLWhenSIGTERMIgnored(t *testing.T) { assert.ErrorIs(t, syscall.Kill(pid, 0), syscall.ESRCH, "SIGTERM-ignoring hypervisor must still be hard-killed") } -func TestKillHypervisorSIGTERMsRunningVGPUHypervisor(t *testing.T) { - markerPath := filepath.Join(t.TempDir(), "terminated") - pid, identity := startTrapProcess(t, "touch "+markerPath+"; exit 0") - socketPath := filepath.Join(t.TempDir(), "missing.sock") - - m := &manager{} - require.NoError(t, m.killHypervisor(context.Background(), &Instance{ - State: StateRunning, - StoredMetadata: StoredMetadata{ - Id: "kill-test", - GPUProfile: "NVIDIA L40S-1Q", - HypervisorProcessIdentity: identity, - SocketPath: socketPath, - }, - })) - - require.Eventually(t, func() bool { - return syscall.Kill(pid, 0) == syscall.ESRCH - }, 5*time.Second, 10*time.Millisecond) - assert.FileExists(t, markerPath, "Running reports true before guest driver init completes, so vGPU hypervisors get SIGTERM in every state") -} - func TestKillHypervisorHardKillsNonVGPUHypervisor(t *testing.T) { markerPath := filepath.Join(t.TempDir(), "terminated") pid, identity := startTrapProcess(t, "touch "+markerPath+"; exit 0") diff --git a/lib/instances/query_test.go b/lib/instances/query_test.go index 3d12c739d..795b00baf 100644 --- a/lib/instances/query_test.go +++ b/lib/instances/query_test.go @@ -42,11 +42,6 @@ func TestListInstancesForReconcileFailsOnInvalidMetadata(t *testing.T) { assert.Equal(t, "valid", listed[0].Id) } -// A concurrent delete can remove an instance between the reconcile listing -// and its metadata load. A vanished record cannot claim a VF, so it must be -// skipped like the release claim scan does — failing instead would zero the -// grace-period retry and silently disable the vendor VFIO sweep whenever it -// races a delete. func TestListInstancesForReconcileSkipsInstanceDeletedDuringListing(t *testing.T) { m := &manager{paths: paths.New(t.TempDir())} @@ -60,9 +55,6 @@ func TestListInstancesForReconcileSkipsInstanceDeletedDuringListing(t *testing.T }})) } - // loadMetadata takes the snapshot-alias read lock, so holding the - // mutation lock parks the reconcile between listing and loading — the - // window a concurrent delete lands in. unlock := hypervisor.LockSnapshotSourceAliasMutation() type result struct { listed []Instance diff --git a/lib/instances/snapshot.go b/lib/instances/snapshot.go index 1d8456ae5..a87acb614 100644 --- a/lib/instances/snapshot.go +++ b/lib/instances/snapshot.go @@ -267,8 +267,6 @@ func (m *manager) restoreSnapshot(ctx context.Context, id string, snapshotID str return nil, fmt.Errorf("%w: cannot restore snapshot while source is %s", ErrInvalidState, sourceInst.State) } if sourceMeta.GPURetainedForCleanup { - // Restoring would rebuild boot config from the snapshot record, whose - // retention flag is false, silently clearing the delete-only marker. return nil, errVGPURetentionStub } diff --git a/lib/instances/snapshot_test.go b/lib/instances/snapshot_test.go index e4fcc048f..31f0a4343 100644 --- a/lib/instances/snapshot_test.go +++ b/lib/instances/snapshot_test.go @@ -67,8 +67,6 @@ func TestCreateSnapshotRejectsVGPURetentionRecord(t *testing.T) { meta.GPURetainedForCleanup = true require.NoError(t, mgr.saveMetadata(meta)) - // The delete-only retention stub has no boot configuration, so a snapshot - // of it could never be restored or forked into a bootable instance. _, err = mgr.CreateSnapshot(ctx, sourceID, CreateSnapshotRequest{ Kind: SnapshotKindStopped, Name: "snapshot-vgpu-retention", @@ -98,8 +96,6 @@ func TestRestoreSnapshotRejectsVGPURetentionRecord(t *testing.T) { meta.GPURetainedForCleanup = true require.NoError(t, mgr.saveMetadata(meta)) - // Restoring into the delete-only stub would rebuild boot config from the - // snapshot record, whose retention flag is false, clearing the marker. _, err = mgr.RestoreSnapshot(ctx, sourceID, snapshot.Id, RestoreSnapshotRequest{ TargetState: StateStopped, TargetHypervisor: mgr.defaultHypervisor, @@ -107,7 +103,6 @@ func TestRestoreSnapshotRejectsVGPURetentionRecord(t *testing.T) { require.ErrorIs(t, err, ErrInvalidState) require.ErrorContains(t, err, "delete it to release the assignment") - // The retained assignment must survive the rejected restore for delete. stored, err := mgr.loadMetadata(sourceID) require.NoError(t, err) assert.True(t, stored.GPURetainedForCleanup) diff --git a/lib/instances/start.go b/lib/instances/start.go index 1089a35a3..8986d7fb3 100644 --- a/lib/instances/start.go +++ b/lib/instances/start.go @@ -67,7 +67,6 @@ func (m *manager) startInstance( } } - // Do not persist the previous VMM's identity with a new vGPU assignment. stored.HypervisorPID = nil stored.HypervisorStartTime = 0 stored.HypervisorBootID = "" @@ -185,9 +184,6 @@ func (m *manager) startInstance( wrapped := fmt.Errorf("create vGPU for profile %s: %w", stored.GPUProfile, err) if saveErr := m.saveMetadata(&retentionMeta); saveErr != nil { log.ErrorContext(ctx, "failed to retain vGPU assignment after create rollback failure", "instance_id", id, "error", saveErr) - // No on-disk record points at the device; retry the release - // in the background instead of waiting for the next startup - // reconcile. m.scheduleOrphanedVGPURelease(ctx, retentionMeta.StoredMetadata) return nil, &VGPUCleanupPendingError{InstanceID: id, Err: fmt.Errorf("%w; retain assignment: %v", wrapped, saveErr)} } diff --git a/lib/instances/storage.go b/lib/instances/storage.go index 40bbba684..1a4d325b0 100644 --- a/lib/instances/storage.go +++ b/lib/instances/storage.go @@ -187,15 +187,10 @@ func removeAllWithRetry(path string, removeAll func(string) error, sleep func(ti } } -// listMetadataFiles returns paths to all instance metadata files, skipping -// entries whose metadata cannot be statted. func (m *manager) listMetadataFiles() ([]string, error) { return m.walkMetadataFiles(false) } -// listMetadataFilesStrict returns paths to all instance metadata files, -// failing on any stat error other than absence, so fail-closed callers see -// an unreadable instance as an error instead of silently missing. func (m *manager) listMetadataFilesStrict() ([]string, error) { return m.walkMetadataFiles(true) } diff --git a/lib/instances/types.go b/lib/instances/types.go index ab41178cd..ed32f5b5d 100644 --- a/lib/instances/types.go +++ b/lib/instances/types.go @@ -151,15 +151,12 @@ type StoredMetadata struct { Devices []string // Device IDs attached to this instance // GPU configuration (vGPU mode) - GPUProfile string // vGPU profile name (e.g., "L40S-1Q") - GPUFramework devices.VGPUFramework - GPUDevicePath string - GPUMdevUUID string // populated for mdev-backed vGPUs - GPUAssignedAt *time.Time // set before hypervisor startup to bound crash recovery protection - // GPURetainedForCleanup marks a delete-only retention stub written when a - // failed create could not release its vGPU: the record has no boot - // configuration, so only delete (which retries the release) may act on it. - GPURetainedForCleanup bool + GPUProfile string // vGPU profile name (e.g., "L40S-1Q") + GPUFramework devices.VGPUFramework + GPUDevicePath string + GPUMdevUUID string // populated for mdev-backed vGPUs + GPUAssignedAt *time.Time // set before hypervisor startup to bound crash recovery protection + GPURetainedForCleanup bool // delete-only stub holding a vGPU assignment // Command overrides (like docker run ) Entrypoint []string // Override image entrypoint (nil = use image default) diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index b82a07d7d..0e09f2f34 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -15,10 +15,7 @@ import ( // persisted hypervisor PID is treated as potentially live. const VGPUAssignmentStartupGracePeriod = 5 * time.Minute -// VGPUCleanupPendingError reports a failed create whose vGPU release also -// failed during rollback. When Retained is true, deleting the retained instance -// retries the release; otherwise a background retry and startup reconciliation -// recover the assignment. +// VGPUCleanupPendingError reports a failed rollback that left a vGPU assigned. type VGPUCleanupPendingError struct { InstanceID string Retained bool @@ -34,9 +31,6 @@ func (e *VGPUCleanupPendingError) Error() string { func (e *VGPUCleanupPendingError) Unwrap() error { return e.Err } -// errVGPURetentionStub rejects every lifecycle verb except delete on a -// retention stub from a failed create: the record has no boot configuration, -// and only delete retries the release of its retained assignment. var errVGPURetentionStub = fmt.Errorf("%w: instance retains a vGPU assignment from a failed create and has no boot configuration; delete it to release the assignment", ErrInvalidState) func (m *manager) createVGPUDevice(ctx context.Context, profileName, instanceID string) (*devices.VGPUDevice, error) { @@ -91,9 +85,6 @@ func clearStoredVGPUDevice(stored *StoredMetadata) { stored.GPUAssignedAt = nil } -// cleanupStartVGPU reports whether the assignment was retained after a failed -// destroy and whether that retention record was persisted, so start can surface -// the pending cleanup as a typed error like create does. func (m *manager) cleanupStartVGPU(ctx context.Context, instanceID string, device *devices.VGPUDevice, assignedAt time.Time, rollbackMeta metadata) (retained, persisted bool) { logger.FromContext(ctx).DebugContext(ctx, "destroying vGPU on cleanup", "instance_id", instanceID, "uuid", device.MdevUUID) assignment := devices.VGPUAssignment{ @@ -124,21 +115,15 @@ func (m *manager) cleanupStartVGPU(ctx context.Context, instanceID string, devic if !retained { return false, false } - // The mid-start save may already have persisted this assignment, so - // delete or a retried start can still release it. if meta, loadErr := m.loadMetadata(instanceID); loadErr == nil && storedVGPUDevicePath(&meta.StoredMetadata) == device.SysfsPath { return true, true } - // No on-disk record points at the device; retry the release in the - // background instead of waiting for the next startup reconcile. m.scheduleOrphanedVGPURelease(ctx, cleanupMeta.StoredMetadata) return true, false } return retained, retained } -// restoreStartMutatedFields must cover every field start mutates before the -// vGPU cleanup runs. func restoreStartMutatedFields(dst, src *StoredMetadata) { dst.HypervisorPID = src.HypervisorPID dst.HypervisorStartTime = src.HypervisorStartTime @@ -162,18 +147,10 @@ func (m *manager) releaseStoredVGPU(ctx context.Context, stored *StoredMetadata) return m.releaseStoredVGPUExcluding(ctx, stored, stored.Id) } -// releaseStoredVGPUExcluding releases stored's assignment while treating -// excludeID's metadata as not a claimant. Callers releasing an instance's own -// persisted assignment exclude that instance; the orphan retry passes no -// exclusion because its instance may have been restarted onto the same VF, -// and that live claim must block the release. func (m *manager) releaseStoredVGPUExcluding(ctx context.Context, stored *StoredMetadata, excludeID string) error { path := storedVGPUDevicePath(stored) if path != "" { - // Vendor VFIO VFs are reused across instances, so the release must - // fail closed on an incomplete inventory. mdev UUIDs are never reused; - // scanning there would let one unreadable metadata file block every - // mdev release on the host. + // Vendor VFIO VFs are reusable, so release fails closed on an incomplete inventory. claimed := false if stored.GPUFramework == devices.VGPUFrameworkVendorVFIO { var err error @@ -201,10 +178,6 @@ func (m *manager) releaseStoredVGPUExcluding(ctx context.Context, stored *Stored return nil } -// vgpuAssignmentClaimedByLiveInstance reports whether another live instance's -// stored metadata claims devicePath. Unreadable metadata, a recent assignment -// without a PID, or unverifiable process ownership returns an error so the -// requester retains its assignment for a later retry. func (m *manager) vgpuAssignmentClaimedByLiveInstance(ctx context.Context, excludeID, devicePath string) (bool, error) { files, err := m.listMetadataFilesStrict() if err != nil { @@ -218,8 +191,6 @@ func (m *manager) vgpuAssignmentClaimedByLiveInstance(ctx context.Context, exclu meta, err := m.loadMetadata(id) if err != nil { if errors.Is(err, ErrNotFound) { - // Deleted between listing and load; a vanished record cannot - // be a live claimant. continue } return false, fmt.Errorf("load metadata for vGPU release check: instance %s: %w", id, err) @@ -262,9 +233,6 @@ func (m *manager) releaseRetainedVGPULocked(ctx context.Context, id string) { } stored := &meta.StoredMetadata if stored.GPURetainedForCleanup { - // Delete-only retention stubs release through delete. Releasing here - // would leave a stub whose start/fork/snapshot errors still claim a - // retained assignment that no longer exists. return } if storedVGPUDevicePath(stored) == "" { diff --git a/lib/instances/vgpu_orphan.go b/lib/instances/vgpu_orphan.go index 513cf9bc9..041402642 100644 --- a/lib/instances/vgpu_orphan.go +++ b/lib/instances/vgpu_orphan.go @@ -8,21 +8,10 @@ import ( ) const ( - // Bounds the retry loop (~10 minutes at the default interval, far beyond - // normal VFIO teardown) so a wedged VF degrades to one operator-actionable - // error instead of indefinite log churn. orphanedVGPUReleaseMaxAttempts = 20 defaultOrphanedVGPUReleaseRetryDelay = 30 * time.Second ) -// scheduleOrphanedVGPURelease retries a vGPU release for an assignment no -// on-disk metadata points at anymore: a release that failed during a -// completed delete (a GPU-busy VMM routinely outlives delete's force-kill -// wait), or a rollback whose retention record could not be saved. Without a -// record, nothing else releases the VF until startup reconciliation. Each -// attempt re-runs the release, so the claim scan and destroy guards apply on -// every retry. The queue is in-memory only; a restart abandons it and startup -// reconciliation sweeps the VF. func (m *manager) scheduleOrphanedVGPURelease(ctx context.Context, stored StoredMetadata) { path := storedVGPUDevicePath(&stored) if path == "" { @@ -43,8 +32,6 @@ func (m *manager) scheduleOrphanedVGPURelease(ctx context.Context, stored Stored if delay <= 0 { delay = defaultOrphanedVGPUReleaseRetryDelay } - // The request context ends with the delete; keep its values for logging - // but detach from its cancellation. go m.retryOrphanedVGPURelease(context.WithoutCancel(ctx), stored, path, delay) } @@ -57,8 +44,6 @@ func (m *manager) retryOrphanedVGPURelease(ctx context.Context, stored StoredMet }() for attempt := 1; attempt <= orphanedVGPUReleaseMaxAttempts; attempt++ { time.Sleep(delay) - // No claim-scan exclusion: unlike delete, a failed start keeps its - // instance record, and a restarted instance may hold this same VF. if err := m.releaseStoredVGPUExcluding(ctx, &stored, ""); err != nil { log.WarnContext(ctx, "orphaned vGPU release retry failed", "instance_id", stored.Id, "device_path", path, "attempt", attempt, "error", err) diff --git a/lib/instances/vgpu_orphan_test.go b/lib/instances/vgpu_orphan_test.go index 25a350064..ee8576a3b 100644 --- a/lib/instances/vgpu_orphan_test.go +++ b/lib/instances/vgpu_orphan_test.go @@ -97,20 +97,6 @@ func TestScheduleOrphanedVGPUReleaseDeduplicatesByDevicePath(t *testing.T) { assert.Equal(t, int32(1), attempts.Load(), "the second schedule for the same path must be dropped") } -func TestScheduleOrphanedVGPUReleaseIgnoresEmptyAssignment(t *testing.T) { - t.Parallel() - - m := &manager{paths: paths.New(t.TempDir())} - m.scheduleOrphanedVGPURelease(context.Background(), StoredMetadata{Id: "no-gpu"}) - - m.orphanedVGPUMu.Lock() - defer m.orphanedVGPUMu.Unlock() - assert.Empty(t, m.orphanedVGPUs) -} - -// TestOrphanedVGPUReleaseReappliesClaimScan pins that the background retry -// goes through releaseStoredVGPU, not a raw destroy: a live claimant found by -// the vendor VFIO claim scan must keep blocking the release on every retry. func TestOrphanedVGPUReleaseReappliesClaimScan(t *testing.T) { t.Parallel() @@ -123,8 +109,6 @@ func TestOrphanedVGPUReleaseReappliesClaimScan(t *testing.T) { return nil }, } - // A claimant with a recent assignment and no persisted PID makes the scan - // fail closed, exactly like the synchronous release path. require.NoError(t, m.ensureDirectories("mid-boot-claimant")) assignedAt := time.Now() require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ diff --git a/lib/instances/vgpu_reconcile.go b/lib/instances/vgpu_reconcile.go index 24be04fe3..aab2da2a1 100644 --- a/lib/instances/vgpu_reconcile.go +++ b/lib/instances/vgpu_reconcile.go @@ -8,8 +8,6 @@ import ( "github.com/kernel/hypeman/lib/logger" ) -// vgpuReconcileFailureRetryDelay spaces retries after a transient metadata or -// device error would otherwise disable orphan recovery until the next restart. const vgpuReconcileFailureRetryDelay = time.Minute func (m *manager) liveVGPUReconcileProtection(ctx context.Context) (map[string]struct{}, time.Duration, error) { @@ -67,8 +65,6 @@ func (m *manager) ReconcileVGPUs(ctx context.Context) { if retryAfter <= 0 { return } - // One pending retry at a time: overlapping calls would fork parallel - // retry chains. if !m.vgpuReconcileRetryPending.CompareAndSwap(false, true) { return } diff --git a/lib/instances/vgpu_reconcile_test.go b/lib/instances/vgpu_reconcile_test.go index 815415d09..03851f40d 100644 --- a/lib/instances/vgpu_reconcile_test.go +++ b/lib/instances/vgpu_reconcile_test.go @@ -65,7 +65,6 @@ func TestReconcileVGPUsRetriesAfterListingFailure(t *testing.T) { require.True(t, m.vgpuReconcileRetryPending.Load(), "a listing failure must schedule a retry instead of disabling the vendor sweep until restart") - // Once the listing recovers, the retry runs the sweep and stops rearming. require.NoError(t, os.Chmod(instanceDir, 0o755)) require.Eventually(t, func() bool { return !m.vgpuReconcileRetryPending.Load() @@ -93,32 +92,3 @@ func TestReconcileVGPUsRetriesAfterDeviceFailure(t *testing.T) { return calls.Load() >= 2 && !m.vgpuReconcileRetryPending.Load() }, 5*time.Second, 10*time.Millisecond) } - -func TestVGPUAssignmentLiveness(t *testing.T) { - now := time.Now().UTC() - recent := now.Add(-time.Minute) - stale := now.Add(-VGPUAssignmentStartupGracePeriod - time.Minute) - pid := 123 - - tests := []struct { - name string - stored StoredMetadata - livePID bool - live bool - remaining time.Duration - }{ - {name: "live PID", stored: StoredMetadata{HypervisorProcessIdentity: HypervisorProcessIdentity{HypervisorPID: &pid}}, livePID: true, live: true}, - {name: "dead PID recent assignment", stored: StoredMetadata{HypervisorProcessIdentity: HypervisorProcessIdentity{HypervisorPID: &pid}, GPUAssignedAt: &recent}, live: true, remaining: VGPUAssignmentStartupGracePeriod - time.Minute}, - {name: "dead PID stale assignment", stored: StoredMetadata{HypervisorProcessIdentity: HypervisorProcessIdentity{HypervisorPID: &pid}, GPUAssignedAt: &stale}}, - {name: "no PID recent assignment", stored: StoredMetadata{GPUAssignedAt: &recent}, live: true, remaining: VGPUAssignmentStartupGracePeriod - time.Minute}, - {name: "no PID stale assignment", stored: StoredMetadata{GPUAssignedAt: &stale}}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - live, remaining := vgpuAssignmentLiveness(&tt.stored, now, tt.livePID) - assert.Equal(t, tt.live, live) - assert.Equal(t, tt.remaining, remaining) - }) - } -} diff --git a/lib/instances/vgpu_retention.go b/lib/instances/vgpu_retention.go index eaca96289..feac72ed2 100644 --- a/lib/instances/vgpu_retention.go +++ b/lib/instances/vgpu_retention.go @@ -65,8 +65,6 @@ func (m *manager) persistVGPURetention(ctx context.Context, retention *vgpuReten if err := m.deleteInstanceData(id); err != nil { log.ErrorContext(ctx, "failed to delete stale instance data after retention failure", "instance_id", id, "error", err) } - // No on-disk record points at the device; retry the release in the - // background instead of waiting for the next startup reconcile. m.scheduleOrphanedVGPURelease(ctx, *retainedVGPU) return false } diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index cb54c509b..4b31bf7c7 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -67,17 +67,13 @@ func TestCleanupFailedCreateRetainsVGPUAssignment(t *testing.T) { assert.Equal(t, stored.GPUDevicePath, retained.GPUDevicePath) assert.Equal(t, stored.GPUMdevUUID, retained.GPUMdevUUID) assert.Equal(t, stored.GPUAssignedAt, retained.GPUAssignedAt) - // Identity fields survive so the retained record lists as a - // recognizable, deletable instance instead of a nameless phantom. assert.Equal(t, stored.Name, retained.Name) assert.Equal(t, stored.GPUProfile, retained.GPUProfile) assert.Equal(t, stored.HypervisorType, retained.HypervisorType) assert.Equal(t, stored.DataDir, retained.DataDir) - // Resource claims released by rollback stay dropped. assert.False(t, retained.NetworkEnabled) assert.Empty(t, retained.IP) assert.Empty(t, retained.Volumes) - // The stub has no boot configuration, so it is marked delete-only. assert.True(t, retained.GPURetainedForCleanup) } @@ -162,19 +158,6 @@ func TestVGPURetentionWrapPending(t *testing.T) { assert.True(t, cleanupPending.Retained) } -func TestVGPUCleanupPendingErrorUnwraps(t *testing.T) { - t.Parallel() - - cause := errors.New("boot failed") - retained := &VGPUCleanupPendingError{InstanceID: "inst-1", Retained: true, Err: cause} - assert.ErrorIs(t, retained, cause) - assert.Equal(t, "boot failed; vGPU release failed during rollback, instance inst-1 retains the assignment", retained.Error()) - - unpersisted := &VGPUCleanupPendingError{InstanceID: "inst-1", Err: cause} - assert.ErrorIs(t, unpersisted, cause) - assert.Equal(t, "boot failed; vGPU release failed during rollback and the retention record for instance inst-1 could not be saved; the release is retried in the background and by the next startup reconcile", unpersisted.Error()) -} - func TestVGPUDevicePendingCleanup(t *testing.T) { t.Parallel() @@ -445,15 +428,12 @@ func TestCleanupStartVGPUReportsRetainedWhenMidStartSaveSurvives(t *testing.T) { } assignedAt := time.Now().UTC() - // The mid-start save already persisted the assignment. meta, err := m.loadMetadata(id) require.NoError(t, err) rollbackMeta := *meta setStoredVGPUDevice(&meta.StoredMetadata, &device, assignedAt) require.NoError(t, m.saveMetadata(meta)) - // The cleanup save fails, but the surviving on-disk record still points - // at the device, so retention must be reported as persisted. instanceDir := filepath.Dir(m.paths.InstanceMetadata(id)) require.NoError(t, os.Chmod(instanceDir, 0o555)) t.Cleanup(func() { _ = os.Chmod(instanceDir, 0o755) }) @@ -603,12 +583,9 @@ func TestVGPUAssignmentClaimedByLiveInstanceGracesRecentDeadPIDClaim(t *testing. GPUAssignedAt: &assignedAt, }})) - // Same bounded grace as startup reconcile: a recent claim whose PID is - // dead fails closed instead of being treated as unclaimed. _, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "requester", "/sys/bus/pci/devices/0000:82:00.4") require.Error(t, err) - // Past the grace period the dead claim no longer blocks the release. stale := assignedAt.Add(-2 * VGPUAssignmentStartupGracePeriod) meta, err := m.loadMetadata(claimantID) require.NoError(t, err) From 78d8fa39a7547032bda748f86be7baa464a42b27 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Tue, 25 Aug 2026 20:32:09 +0000 Subject: [PATCH 64/76] Track vendor VFIO assignment age and owner for periodic sweeps Record the owning instance and assignment time for each vendor VFIO VF so a reconcile sweep can run while instances are being created: recently assigned VFs get a grace period before they are eligible (mirroring orphanedMdevGracePeriod), and owned VFs are destroyed with their recorded owner ID instead of failing the ownership check. --- lib/devices/vendor_vfio_linux.go | 34 +++++++++++++++++++------ lib/devices/vendor_vfio_linux_test.go | 36 ++++++++++++++++++++++++--- 2 files changed, 59 insertions(+), 11 deletions(-) diff --git a/lib/devices/vendor_vfio_linux.go b/lib/devices/vendor_vfio_linux.go index 2c576b4d2..74eae4948 100644 --- a/lib/devices/vendor_vfio_linux.go +++ b/lib/devices/vendor_vfio_linux.go @@ -7,6 +7,7 @@ import ( "errors" "fmt" "log/slog" + "maps" "os" "path/filepath" "sort" @@ -14,6 +15,7 @@ import ( "strings" "sync" "syscall" + "time" "github.com/kernel/hypeman/lib/logger" ) @@ -21,13 +23,23 @@ import ( const ( pciDevicesPath = "/sys/bus/pci/devices" vfioDevicesPath = "/dev/vfio/devices" + + // vendorVFIOAssignmentGracePeriod protects assignments created by this + // process from the periodic sweep until their owning instance has had time + // to persist metadata and boot, mirroring orphanedMdevGracePeriod. + vendorVFIOAssignmentGracePeriod = 5 * time.Minute ) +type vendorVFIOOwner struct { + instanceID string + assignedAt time.Time +} + type vendorVFIOSysfs struct { pciDevicesPath string procPath string vfioDevicesPath string - owners map[string]string + owners map[string]vendorVFIOOwner framebufferByType map[string]int openVFIOPathsFunc func() (map[string]struct{}, error) } @@ -37,7 +49,7 @@ var ( pciDevicesPath: pciDevicesPath, procPath: procPath, vfioDevicesPath: vfioDevicesPath, - owners: make(map[string]string), + owners: make(map[string]vendorVFIOOwner), framebufferByType: make(map[string]int), } vendorVFIOMu sync.Mutex @@ -196,7 +208,7 @@ func (s vendorVFIOSysfs) create(ctx context.Context, profileName, instanceID str verifyErr := fmt.Errorf("verify vGPU on VF %s: type is %s, want %s", targetVF, currentType, requested.TypeName) return nil, s.rollbackCreate(currentTypePath, targetVF, instanceID, device, verifyErr) } - s.owners[targetVF] = instanceID + s.owners[targetVF] = vendorVFIOOwner{instanceID: instanceID, assignedAt: time.Now()} logger.FromContext(ctx).InfoContext(ctx, "created vendor VFIO vGPU", "profile", profileName, @@ -229,10 +241,10 @@ func (s vendorVFIOSysfs) destroy(ctx context.Context, vfAddress, instanceID stri if instanceID == "" { return fmt.Errorf("cannot release vendor VFIO vGPU on VF %s without instance ID", vfAddress) } - if owner != instanceID { + if owner.instanceID != instanceID { log.WarnContext(ctx, "skipping vendor VFIO vGPU release owned by another instance", "vf", vfAddress, - "owner_instance_id", owner, + "owner_instance_id", owner.instanceID, "requesting_instance_id", instanceID, ) return nil @@ -264,6 +276,9 @@ func (s vendorVFIOSysfs) reconcile(ctx context.Context, protectedDevicePaths map if err != nil { return err } + vendorVFIOMu.Lock() + owners := maps.Clone(s.owners) + vendorVFIOMu.Unlock() log := logger.FromContext(ctx) protectedVFs := make(map[string]struct{}, len(protectedDevicePaths)) for path := range protectedDevicePaths { @@ -278,6 +293,11 @@ func (s vendorVFIOSysfs) reconcile(ctx context.Context, protectedDevicePaths map log.DebugContext(ctx, "skipping vendor VFIO vGPU held by a live instance", "vf", vf.PCIAddress) continue } + owner := owners[vf.PCIAddress] + if !owner.assignedAt.IsZero() && time.Since(owner.assignedAt) < vendorVFIOAssignmentGracePeriod { + log.DebugContext(ctx, "skipping recently assigned vendor VFIO vGPU during grace period", "vf", vf.PCIAddress) + continue + } if openPaths == nil { if openPaths, err = s.openVFIOPaths(); err != nil { return fmt.Errorf("scan open VFIO handles: %w", err) @@ -292,7 +312,7 @@ func (s vendorVFIOSysfs) reconcile(ctx context.Context, protectedDevicePaths map log.WarnContext(ctx, "preserving vendor VFIO vGPU held open without a live instance claim", "vf", vf.PCIAddress) continue } - if err := s.destroy(ctx, vf.PCIAddress, ""); err != nil { + if err := s.destroy(ctx, vf.PCIAddress, owner.instanceID); err != nil { log.WarnContext(ctx, "failed to destroy orphaned vendor VFIO vGPU", "vf", vf.PCIAddress, "error", err) } } @@ -514,7 +534,7 @@ func framebufferFromProfileName(name string) int { func (s vendorVFIOSysfs) rollbackCreate(currentTypePath, vfAddress, instanceID string, device VGPUDevice, verifyErr error) error { if err := os.WriteFile(currentTypePath, []byte("0"), 0200); err != nil { - s.owners[vfAddress] = instanceID + s.owners[vfAddress] = vendorVFIOOwner{instanceID: instanceID, assignedAt: time.Now()} return &VGPUCreateCleanupPendingError{ Device: device, Err: errors.Join(verifyErr, fmt.Errorf("roll back vGPU on VF %s: %w", vfAddress, err)), diff --git a/lib/devices/vendor_vfio_linux_test.go b/lib/devices/vendor_vfio_linux_test.go index bd7293922..f7b0d6c3f 100644 --- a/lib/devices/vendor_vfio_linux_test.go +++ b/lib/devices/vendor_vfio_linux_test.go @@ -8,6 +8,7 @@ import ( "os" "path/filepath" "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -372,6 +373,32 @@ func TestVendorVFIOReconcile(t *testing.T) { assertFileValue(t, filepath.Join(sysfs.pciDevicesPath, "0000:e3:00.4", "nvidia", "current_vgpu_type"), "1148") } +func TestVendorVFIOReconcileSkipsRecentlyAssignedVF(t *testing.T) { + t.Parallel() + + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "1148", "") + sysfs.owners["0000:82:00.4"] = vendorVFIOOwner{instanceID: "mid-create", assignedAt: time.Now()} + + require.NoError(t, sysfs.reconcile(context.Background(), nil)) + assertFileValue(t, filepath.Join(sysfs.pciDevicesPath, "0000:82:00.4", "nvidia", "current_vgpu_type"), "1148") +} + +func TestVendorVFIOReconcileDestroysOwnedVFPastGracePeriod(t *testing.T) { + t.Parallel() + + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "1148", "") + sysfs.owners["0000:82:00.4"] = vendorVFIOOwner{ + instanceID: "deleted-instance", + assignedAt: time.Now().Add(-vendorVFIOAssignmentGracePeriod - time.Minute), + } + + require.NoError(t, sysfs.reconcile(context.Background(), nil)) + assertFileValue(t, filepath.Join(sysfs.pciDevicesPath, "0000:82:00.4", "nvidia", "current_vgpu_type"), "0") + assert.Empty(t, sysfs.owners) +} + func TestVendorVFIOReconcileRechecksOpenHandlesBeforeDestroy(t *testing.T) { t.Parallel() @@ -518,7 +545,7 @@ func TestRollbackVendorVFIOCreate(t *testing.T) { t.Run("preserves verification error", func(t *testing.T) { currentTypePath := filepath.Join(t.TempDir(), "current_vgpu_type") require.NoError(t, os.WriteFile(currentTypePath, []byte("1148"), 0644)) - sysfs := vendorVFIOSysfs{owners: make(map[string]string)} + sysfs := vendorVFIOSysfs{owners: make(map[string]vendorVFIOOwner)} err := sysfs.rollbackCreate(currentTypePath, device.VFAddress, "instance-1", device, verifyErr) require.ErrorIs(t, err, verifyErr) @@ -528,7 +555,7 @@ func TestRollbackVendorVFIOCreate(t *testing.T) { t.Run("retains assignment when rollback fails", func(t *testing.T) { currentTypePath := filepath.Join(t.TempDir(), "missing", "current_vgpu_type") - sysfs := vendorVFIOSysfs{owners: make(map[string]string)} + sysfs := vendorVFIOSysfs{owners: make(map[string]vendorVFIOOwner)} err := sysfs.rollbackCreate(currentTypePath, device.VFAddress, "instance-1", device, verifyErr) require.ErrorIs(t, err, verifyErr) @@ -536,7 +563,8 @@ func TestRollbackVendorVFIOCreate(t *testing.T) { var pending *VGPUCreateCleanupPendingError require.ErrorAs(t, err, &pending) assert.Equal(t, device, pending.Device) - assert.Equal(t, "instance-1", sysfs.owners[device.VFAddress]) + assert.Equal(t, "instance-1", sysfs.owners[device.VFAddress].instanceID) + assert.False(t, sysfs.owners[device.VFAddress].assignedAt.IsZero()) }) } @@ -566,7 +594,7 @@ func newTestVendorVFIOSysfs(t *testing.T) testVendorVFIOSysfs { pciDevicesPath: pci, procPath: proc, vfioDevicesPath: vfio, - owners: make(map[string]string), + owners: make(map[string]vendorVFIOOwner), framebufferByType: make(map[string]int), }} } From 937caf1f66ec10ad9d35535326ac9a40e33c2545 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Tue, 25 Aug 2026 20:32:16 +0000 Subject: [PATCH 65/76] Replace vGPU release retry machinery with a periodic reconciler Run the fail-closed vGPU reconcile once at startup and every minute after, skipping hosts without a vGPU framework. Each pass retries releases for assignments whose owner is no longer live (re-verified under the instance lock) and then sweeps device-level leftovers with no live metadata claim. This deletes the per-path orphan retry goroutines - whose path-keyed dedup could drop cleanup for a newer assignment reusing the same VF - the CAS/timer retry in ReconcileVGPUs, the stopped-instance release special case in StopInstance, the retention fallbacks that scheduled background retries, and the orphan-abandoned metric. --- cmd/api/main.go | 4 +- lib/devices/GPU.md | 2 +- lib/instances/delete.go | 3 +- lib/instances/lifecycle_noop_test.go | 44 ++---- lib/instances/manager.go | 17 +- lib/instances/metrics.go | 17 -- lib/instances/start.go | 3 +- lib/instances/vgpu.go | 30 +--- lib/instances/vgpu_orphan.go | 59 ------- lib/instances/vgpu_orphan_test.go | 130 --------------- lib/instances/vgpu_reconcile.go | 142 +++++++++++------ lib/instances/vgpu_reconcile_test.go | 226 +++++++++++++++++++++++---- lib/instances/vgpu_retention.go | 3 +- lib/instances/vgpu_test.go | 12 +- 14 files changed, 321 insertions(+), 371 deletions(-) delete mode 100644 lib/instances/vgpu_orphan.go delete mode 100644 lib/instances/vgpu_orphan_test.go diff --git a/cmd/api/main.go b/cmd/api/main.go index 3bd0fff39..faa9ac15a 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -385,8 +385,8 @@ func run() error { } logger.Info("Reconciling vGPU devices...") - if r, ok := app.InstanceManager.(interface{ ReconcileVGPUs(context.Context) }); ok { - r.ReconcileVGPUs(ctx) + if r, ok := app.InstanceManager.(interface{ StartVGPUReconciler(context.Context) }); ok { + r.StartVGPUReconciler(ctx) } // Wire up resource validator for aggregate limit checking diff --git a/lib/devices/GPU.md b/lib/devices/GPU.md index ff11c5cde..d04dcf599 100644 --- a/lib/devices/GPU.md +++ b/lib/devices/GPU.md @@ -97,7 +97,7 @@ Instance Create → Assign profile to VF → Attach VF to VM → Instance Runnin Instance Stop/Delete → Release profile → VF available again ``` -Hypeman reconciles orphaned assignments on server restart while preserving devices held open by a running VMM. A release that fails during delete (typically because a GPU-busy VMM's kernel-side VFIO teardown outlives the force-kill wait) is retried in the background for up to ten minutes, so a completed delete does not strand the VF until the next restart. +Hypeman reconciles orphaned assignments with a periodic fail-closed pass: once at startup and every minute afterward (skipped entirely on hosts without GPUs). Each pass releases assignments whose owning instance is no longer live and clears their metadata, then sweeps device-level leftovers with no live metadata claim. Devices held open by a running VMM and assignments younger than five minutes are preserved, so a release that fails during stop or delete (typically because a GPU-busy VMM's kernel-side VFIO teardown outlives the force-kill wait) is simply retried on later passes until the device is free. ### Hypervisor Support diff --git a/lib/instances/delete.go b/lib/instances/delete.go index 160ae807d..d2f0e63d9 100644 --- a/lib/instances/delete.go +++ b/lib/instances/delete.go @@ -146,8 +146,7 @@ func (m *manager) deleteInstanceWithOptions( log.InfoContext(ctx, "destroying vGPU", "instance_id", id, "uuid", stored.GPUMdevUUID) } if err := m.releaseStoredVGPU(ctx, stored); err != nil { - log.WarnContext(ctx, "failed to destroy vGPU, continuing with cleanup", "instance_id", id, "uuid", stored.GPUMdevUUID, "error", err) - m.scheduleOrphanedVGPURelease(ctx, *stored) + log.WarnContext(ctx, "failed to destroy vGPU, continuing with cleanup; the periodic vGPU reconcile releases it once free", "instance_id", id, "uuid", stored.GPUMdevUUID, "error", err) } else if hadVGPUAssignment { if err := m.saveMetadata(meta); err != nil { log.WarnContext(ctx, "failed to save metadata after vGPU release", "instance_id", id, "error", err) diff --git a/lib/instances/lifecycle_noop_test.go b/lib/instances/lifecycle_noop_test.go index 90f2e7aa1..1843485e3 100644 --- a/lib/instances/lifecycle_noop_test.go +++ b/lib/instances/lifecycle_noop_test.go @@ -152,7 +152,6 @@ func TestLifecycleNoopStandbyWithOptionsStillRejectsStandbyInstance(t *testing.T func TestDeleteContinuesWhenVGPUReleaseFails(t *testing.T) { m, id := newLifecycleNoopManagerWithInstance(t, StateStopped, time.Now().UTC()) - m.orphanedVGPURetryDelay = time.Millisecond meta, err := m.loadMetadata(id) require.NoError(t, err) meta.GPUProfile = "NVIDIA L40S-2Q" @@ -162,9 +161,8 @@ func TestDeleteContinuesWhenVGPUReleaseFails(t *testing.T) { // A failed release is logged and the delete continues, matching the // pre-refactor contract; the leaked assignment is recovered by the - // background retry or startup reconciliation. + // periodic vGPU reconcile. require.NoError(t, m.DeleteInstance(context.Background(), id)) - waitForOrphanQueueEmpty(t, m) _, err = m.loadMetadata(id) require.Error(t, err, "instance data must be deleted despite the failed release") @@ -280,7 +278,6 @@ func TestDeleteDropsStaleVGPUClaimedByLiveInstance(t *testing.T) { func TestDeleteContinuesTeardownAfterFailedVGPURelease(t *testing.T) { m, id := newLifecycleNoopManagerWithInstance(t, StateStopped, time.Now().UTC()) - m.orphanedVGPURetryDelay = time.Millisecond deviceManager := &recordingDeviceManager{} m.deviceManager = deviceManager meta, err := m.loadMetadata(id) @@ -294,7 +291,6 @@ func TestDeleteContinuesTeardownAfterFailedVGPURelease(t *testing.T) { // The failed release must not block the rest of the teardown: devices // are detached and the instance is fully deleted. require.NoError(t, m.DeleteInstance(context.Background(), id)) - waitForOrphanQueueEmpty(t, m) assert.Equal(t, []string{"dev-1"}, deviceManager.detached) _, err = m.loadMetadata(id) @@ -343,8 +339,9 @@ func TestStartRejectsVGPURetentionRecord(t *testing.T) { assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", stored.GPUDevicePath) } -func TestStopStoppedInstanceReleasesRetainedVGPU(t *testing.T) { +func TestReconcileReleasesRetainedVGPUOnStoppedInstance(t *testing.T) { m, id := newLifecycleNoopManagerWithInstance(t, StateStopped, time.Now().UTC()) + m.reconcileVGPUDevices = func(context.Context, map[string]struct{}, bool) error { return nil } meta, err := m.loadMetadata(id) require.NoError(t, err) meta.GPUProfile = "NVIDIA L40S-2Q" @@ -352,38 +349,26 @@ func TestStopStoppedInstanceReleasesRetainedVGPU(t *testing.T) { meta.GPUDevicePath = "/sys/bus/pci/devices/0000:82:00.4" require.NoError(t, m.saveMetadata(meta)) + // Stop on an already-stopped instance is a no-op for the assignment; the + // periodic reconcile retries the release. inst, err := m.StopInstance(context.Background(), id) require.NoError(t, err) require.NotNil(t, inst) assert.Equal(t, StateStopped, inst.State) - stored, err := m.loadMetadata(id) require.NoError(t, err) - assert.Empty(t, stored.GPUDevicePath) -} - -func TestStopStoppedInstanceLeavesRetentionStubForDelete(t *testing.T) { - m, id := newLifecycleNoopManagerWithInstance(t, StateStopped, time.Now().UTC()) - meta, err := m.loadMetadata(id) - require.NoError(t, err) - meta.GPUProfile = "NVIDIA L40S-2Q" - meta.GPUFramework = devices.VGPUFrameworkNone - meta.GPUDevicePath = "/sys/bus/pci/devices/0000:82:00.4" - meta.GPURetainedForCleanup = true - require.NoError(t, m.saveMetadata(meta)) - - inst, err := m.StopInstance(context.Background(), id) - require.NoError(t, err) - require.NotNil(t, inst) + assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", stored.GPUDevicePath) - stored, err := m.loadMetadata(id) + m.ReconcileVGPUs(context.Background()) + stored, err = m.loadMetadata(id) require.NoError(t, err) - assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", stored.GPUDevicePath) - assert.True(t, stored.GPURetainedForCleanup) + assert.Empty(t, stored.GPUDevicePath) + assert.Equal(t, "NVIDIA L40S-2Q", stored.GPUProfile, "profile is kept for the next start") } -func TestStopStoppedInstanceVGPUReleaseFailureRemainsNoop(t *testing.T) { +func TestReconcileVGPUReleaseFailureKeepsStoppedInstanceUsable(t *testing.T) { m, id := newLifecycleNoopManagerWithInstance(t, StateStopped, time.Now().UTC()) + m.reconcileVGPUDevices = func(context.Context, map[string]struct{}, bool) error { return nil } meta, err := m.loadMetadata(id) require.NoError(t, err) meta.GPUProfile = "NVIDIA L40S-2Q" @@ -391,10 +376,7 @@ func TestStopStoppedInstanceVGPUReleaseFailureRemainsNoop(t *testing.T) { meta.GPUDevicePath = "/sys/bus/pci/devices/0000:82:00.4" require.NoError(t, m.saveMetadata(meta)) - inst, err := m.StopInstance(context.Background(), id) - require.NoError(t, err) - require.NotNil(t, inst) - assert.Equal(t, StateStopped, inst.State) + m.ReconcileVGPUs(context.Background()) stored, err := m.loadMetadata(id) require.NoError(t, err) diff --git a/lib/instances/manager.go b/lib/instances/manager.go index 7cd94262a..8c298ed24 100644 --- a/lib/instances/manager.go +++ b/lib/instances/manager.go @@ -8,7 +8,6 @@ import ( "path/filepath" "strings" "sync" - "sync/atomic" "time" "github.com/kernel/hypeman/lib/devices" @@ -215,12 +214,10 @@ type manager struct { // Periodic TAP garbage collection reconciler. tapGCOnce sync.Once - orphanedVGPUMu sync.Mutex - orphanedVGPUs map[string]struct{} - orphanedVGPURetryDelay time.Duration - - vgpuReconcileRetryPending atomic.Bool - vgpuReconcileRetryDelay time.Duration + // Periodic vGPU reconciler. + vgpuReconcileOnce sync.Once + vgpuReconcileInterval time.Duration + discoverVGPU func() (devices.VGPUFramework, []devices.VirtualFunction, error) vgpuInitTermGrace time.Duration @@ -665,12 +662,6 @@ func (m *manager) StopInstance(ctx context.Context, id string) (*Instance, error if err := m.markRestartManualStopLocked(ctx, id); err != nil { return nil, err } - // A stopped instance can retain a vGPU assignment when the release - // failed during the original stop. Retry it here so the vGPU slot is - // not held until the next start, delete, or hypeman restart. A failed - // retry only logs, keeping stop's no-op contract for already-stopped - // instances. - m.releaseRetainedVGPULocked(ctx, id) updated, err := m.currentInstanceWithoutHydration(ctx, id) if err != nil { return nil, err diff --git a/lib/instances/metrics.go b/lib/instances/metrics.go index de71c2686..1ada5ac1e 100644 --- a/lib/instances/metrics.go +++ b/lib/instances/metrics.go @@ -94,7 +94,6 @@ type Metrics struct { lifecycleEventsDroppedTotal metric.Int64Counter forkMemFileShareFallbacksTotal metric.Int64Counter ttlReaperDeletionsTotal metric.Int64Counter - vgpuOrphanReleasesAbandonedTotal metric.Int64Counter tracer trace.Tracer } @@ -271,14 +270,6 @@ func newInstanceMetrics(meter metric.Meter, tracer trace.Tracer, m *manager) (*M return nil, err } - vgpuOrphanReleasesAbandonedTotal, err := meter.Int64Counter( - "hypeman_instances_vgpu_orphan_releases_abandoned_total", - metric.WithDescription("Total orphaned vGPU release retries that gave up, leaving the VF allocated until startup reconciliation or manual remediation"), - ) - if err != nil { - return nil, err - } - // Register observable gauge for instance counts by state instancesTotal, err := meter.Int64ObservableGauge( "hypeman_instances_total", @@ -473,7 +464,6 @@ func newInstanceMetrics(meter metric.Meter, tracer trace.Tracer, m *manager) (*M lifecycleEventsDroppedTotal: lifecycleEventsDroppedTotal, forkMemFileShareFallbacksTotal: forkMemFileShareFallbacksTotal, ttlReaperDeletionsTotal: ttlReaperDeletionsTotal, - vgpuOrphanReleasesAbandonedTotal: vgpuOrphanReleasesAbandonedTotal, tracer: tracer, }, nil } @@ -573,13 +563,6 @@ func (m *manager) recordTimeToRunning(ctx context.Context, stored *StoredMetadat m.metrics.timeToRunning.Record(ctx, duration, metric.WithAttributes(attrs...)) } -func (m *manager) recordVGPUOrphanReleaseAbandoned(ctx context.Context) { - if m.metrics == nil { - return - } - m.metrics.vgpuOrphanReleasesAbandonedTotal.Add(ctx, 1) -} - // recordStateTransition records a state transition with hypervisor label. func (m *manager) recordStateTransition(ctx context.Context, fromState, toState string, hvType hypervisor.Type) { if m.metrics == nil { diff --git a/lib/instances/start.go b/lib/instances/start.go index 8986d7fb3..19bb64697 100644 --- a/lib/instances/start.go +++ b/lib/instances/start.go @@ -55,7 +55,7 @@ func (m *manager) startInstance( // Release any assignment retained by an earlier failed release and // persist the cleared fields immediately, so a failure later in start // cannot leave on-disk metadata pointing at a device that is already - // gone (matching releaseRetainedVGPULocked). + // gone. if storedVGPUDevicePath(stored) != "" { if err := m.releaseStoredVGPU(ctx, stored); err != nil { log.ErrorContext(ctx, "failed to release stale vGPU before start", "instance_id", id, "error", err) @@ -184,7 +184,6 @@ func (m *manager) startInstance( wrapped := fmt.Errorf("create vGPU for profile %s: %w", stored.GPUProfile, err) if saveErr := m.saveMetadata(&retentionMeta); saveErr != nil { log.ErrorContext(ctx, "failed to retain vGPU assignment after create rollback failure", "instance_id", id, "error", saveErr) - m.scheduleOrphanedVGPURelease(ctx, retentionMeta.StoredMetadata) return nil, &VGPUCleanupPendingError{InstanceID: id, Err: fmt.Errorf("%w; retain assignment: %v", wrapped, saveErr)} } return nil, &VGPUCleanupPendingError{InstanceID: id, Retained: true, Err: wrapped} diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index 0e09f2f34..3fb26778a 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -26,7 +26,7 @@ func (e *VGPUCleanupPendingError) Error() string { if e.Retained { return fmt.Sprintf("%v; vGPU release failed during rollback, instance %s retains the assignment", e.Err, e.InstanceID) } - return fmt.Sprintf("%v; vGPU release failed during rollback and the retention record for instance %s could not be saved; the release is retried in the background and by the next startup reconcile", e.Err, e.InstanceID) + return fmt.Sprintf("%v; vGPU release failed during rollback and the retention record for instance %s could not be saved; the periodic vGPU reconcile retries the release", e.Err, e.InstanceID) } func (e *VGPUCleanupPendingError) Unwrap() error { return e.Err } @@ -118,7 +118,6 @@ func (m *manager) cleanupStartVGPU(ctx context.Context, instanceID string, devic if meta, loadErr := m.loadMetadata(instanceID); loadErr == nil && storedVGPUDevicePath(&meta.StoredMetadata) == device.SysfsPath { return true, true } - m.scheduleOrphanedVGPURelease(ctx, cleanupMeta.StoredMetadata) return true, false } return retained, retained @@ -220,33 +219,6 @@ func (m *manager) vgpuAssignmentClaimedByLiveInstance(ctx context.Context, exclu return false, nil } -// releaseRetainedVGPULocked releases a vGPU assignment retained on a stopped -// instance after a failed release during the original stop. It is a no-op -// when no assignment is retained, and a failed retry only logs so the -// metadata stays for the next retry. The caller must hold the instance lock. -func (m *manager) releaseRetainedVGPULocked(ctx context.Context, id string) { - log := logger.FromContext(ctx) - meta, err := m.loadMetadata(id) - if err != nil { - log.WarnContext(ctx, "failed to load metadata for retained vGPU release", "instance_id", id, "error", err) - return - } - stored := &meta.StoredMetadata - if stored.GPURetainedForCleanup { - return - } - if storedVGPUDevicePath(stored) == "" { - return - } - if err := m.releaseStoredVGPU(ctx, stored); err != nil { - log.WarnContext(ctx, "failed to destroy retained vGPU; retaining assignment metadata", "instance_id", id, "error", err) - return - } - if err := m.saveMetadata(meta); err != nil { - log.WarnContext(ctx, "failed to save metadata after retained vGPU release", "instance_id", id, "error", err) - } -} - func storedVGPUDevicePath(stored *StoredMetadata) string { if stored.GPUDevicePath != "" { return stored.GPUDevicePath diff --git a/lib/instances/vgpu_orphan.go b/lib/instances/vgpu_orphan.go deleted file mode 100644 index 041402642..000000000 --- a/lib/instances/vgpu_orphan.go +++ /dev/null @@ -1,59 +0,0 @@ -package instances - -import ( - "context" - "time" - - "github.com/kernel/hypeman/lib/logger" -) - -const ( - orphanedVGPUReleaseMaxAttempts = 20 - defaultOrphanedVGPUReleaseRetryDelay = 30 * time.Second -) - -func (m *manager) scheduleOrphanedVGPURelease(ctx context.Context, stored StoredMetadata) { - path := storedVGPUDevicePath(&stored) - if path == "" { - return - } - m.orphanedVGPUMu.Lock() - if m.orphanedVGPUs == nil { - m.orphanedVGPUs = make(map[string]struct{}) - } - if _, pending := m.orphanedVGPUs[path]; pending { - m.orphanedVGPUMu.Unlock() - return - } - m.orphanedVGPUs[path] = struct{}{} - m.orphanedVGPUMu.Unlock() - - delay := m.orphanedVGPURetryDelay - if delay <= 0 { - delay = defaultOrphanedVGPUReleaseRetryDelay - } - go m.retryOrphanedVGPURelease(context.WithoutCancel(ctx), stored, path, delay) -} - -func (m *manager) retryOrphanedVGPURelease(ctx context.Context, stored StoredMetadata, path string, delay time.Duration) { - log := logger.FromContext(ctx) - defer func() { - m.orphanedVGPUMu.Lock() - delete(m.orphanedVGPUs, path) - m.orphanedVGPUMu.Unlock() - }() - for attempt := 1; attempt <= orphanedVGPUReleaseMaxAttempts; attempt++ { - time.Sleep(delay) - if err := m.releaseStoredVGPUExcluding(ctx, &stored, ""); err != nil { - log.WarnContext(ctx, "orphaned vGPU release retry failed", - "instance_id", stored.Id, "device_path", path, "attempt", attempt, "error", err) - continue - } - log.InfoContext(ctx, "released orphaned vGPU after delete", - "instance_id", stored.Id, "device_path", path, "attempt", attempt) - return - } - m.recordVGPUOrphanReleaseAbandoned(ctx) - log.ErrorContext(ctx, "giving up on orphaned vGPU release; VF stays allocated until startup reconciliation or manual remediation", - "instance_id", stored.Id, "device_path", path, "attempts", orphanedVGPUReleaseMaxAttempts) -} diff --git a/lib/instances/vgpu_orphan_test.go b/lib/instances/vgpu_orphan_test.go deleted file mode 100644 index ee8576a3b..000000000 --- a/lib/instances/vgpu_orphan_test.go +++ /dev/null @@ -1,130 +0,0 @@ -package instances - -import ( - "context" - "errors" - "sync/atomic" - "testing" - "time" - - "github.com/kernel/hypeman/lib/devices" - "github.com/kernel/hypeman/lib/paths" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func waitForOrphanQueueEmpty(t *testing.T, m *manager) { - t.Helper() - require.Eventually(t, func() bool { - m.orphanedVGPUMu.Lock() - defer m.orphanedVGPUMu.Unlock() - return len(m.orphanedVGPUs) == 0 - }, 5*time.Second, 5*time.Millisecond, "orphan retry should finish and clear its queue entry") -} - -func TestScheduleOrphanedVGPUReleaseRetriesUntilSuccess(t *testing.T) { - t.Parallel() - - var attempts atomic.Int32 - m := &manager{ - paths: paths.New(t.TempDir()), - orphanedVGPURetryDelay: time.Millisecond, - destroyVGPU: func(context.Context, devices.VGPUAssignment) error { - if attempts.Add(1) < 3 { - return errors.New("operation not permitted") - } - return nil - }, - } - m.scheduleOrphanedVGPURelease(context.Background(), StoredMetadata{ - Id: "deleted-instance", - GPUFramework: devices.VGPUFrameworkVendorVFIO, - GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", - }) - - waitForOrphanQueueEmpty(t, m) - assert.Equal(t, int32(3), attempts.Load(), "release should succeed on the third attempt and stop retrying") -} - -func TestScheduleOrphanedVGPUReleaseGivesUpAfterMaxAttempts(t *testing.T) { - t.Parallel() - - var attempts atomic.Int32 - m := &manager{ - paths: paths.New(t.TempDir()), - orphanedVGPURetryDelay: time.Millisecond, - destroyVGPU: func(context.Context, devices.VGPUAssignment) error { - attempts.Add(1) - return errors.New("vGPU destroy failed: 0xffffffff") - }, - } - m.scheduleOrphanedVGPURelease(context.Background(), StoredMetadata{ - Id: "deleted-instance", - GPUFramework: devices.VGPUFrameworkVendorVFIO, - GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", - }) - - waitForOrphanQueueEmpty(t, m) - assert.Equal(t, int32(orphanedVGPUReleaseMaxAttempts), attempts.Load(), - "a wedged VF should get exactly the bounded number of attempts") -} - -func TestScheduleOrphanedVGPUReleaseDeduplicatesByDevicePath(t *testing.T) { - t.Parallel() - - var attempts atomic.Int32 - release := make(chan struct{}) - m := &manager{ - paths: paths.New(t.TempDir()), - orphanedVGPURetryDelay: 20 * time.Millisecond, - destroyVGPU: func(context.Context, devices.VGPUAssignment) error { - attempts.Add(1) - <-release - return nil - }, - } - stored := StoredMetadata{ - Id: "deleted-instance", - GPUFramework: devices.VGPUFrameworkVendorVFIO, - GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", - } - m.scheduleOrphanedVGPURelease(context.Background(), stored) - m.scheduleOrphanedVGPURelease(context.Background(), stored) - - require.Eventually(t, func() bool { return attempts.Load() == 1 }, 5*time.Second, 5*time.Millisecond) - close(release) - waitForOrphanQueueEmpty(t, m) - assert.Equal(t, int32(1), attempts.Load(), "the second schedule for the same path must be dropped") -} - -func TestOrphanedVGPUReleaseReappliesClaimScan(t *testing.T) { - t.Parallel() - - var destroys atomic.Int32 - m := &manager{ - paths: paths.New(t.TempDir()), - orphanedVGPURetryDelay: time.Millisecond, - destroyVGPU: func(context.Context, devices.VGPUAssignment) error { - destroys.Add(1) - return nil - }, - } - require.NoError(t, m.ensureDirectories("mid-boot-claimant")) - assignedAt := time.Now() - require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ - Id: "mid-boot-claimant", - Name: "mid-boot-claimant", - GPUFramework: devices.VGPUFrameworkVendorVFIO, - GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", - GPUAssignedAt: &assignedAt, - }})) - - m.scheduleOrphanedVGPURelease(context.Background(), StoredMetadata{ - Id: "deleted-instance", - GPUFramework: devices.VGPUFrameworkVendorVFIO, - GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", - }) - - waitForOrphanQueueEmpty(t, m) - assert.Zero(t, destroys.Load(), "no destroy may fire while the claim scan cannot clear the path") -} diff --git a/lib/instances/vgpu_reconcile.go b/lib/instances/vgpu_reconcile.go index aab2da2a1..966bf1118 100644 --- a/lib/instances/vgpu_reconcile.go +++ b/lib/instances/vgpu_reconcile.go @@ -2,51 +2,62 @@ package instances import ( "context" + "errors" "time" "github.com/kernel/hypeman/lib/devices" "github.com/kernel/hypeman/lib/logger" ) -const vgpuReconcileFailureRetryDelay = time.Minute +const defaultVGPUReconcileInterval = time.Minute -func (m *manager) liveVGPUReconcileProtection(ctx context.Context) (map[string]struct{}, time.Duration, error) { - allInstances, err := m.listInstancesForReconcile(ctx) +// StartVGPUReconciler runs one reconcile pass and then keeps reconciling +// periodically until ctx is cancelled. Hosts without a vGPU framework skip +// reconciliation entirely. A discovery failure starts the reconciler anyway: +// a transient sysfs error must not disable cleanup on a GPU host. +func (m *manager) StartVGPUReconciler(ctx context.Context) { + discover := m.discoverVGPU + if discover == nil { + discover = devices.DiscoverVGPU + } + framework, _, err := discover() + if err == nil && framework == devices.VGPUFrameworkNone { + return + } if err != nil { - return nil, 0, err + logger.FromContext(ctx).WarnContext(ctx, "failed to discover vGPU framework; starting vGPU reconciler anyway", "error", err) } - protected := make(map[string]struct{}) - var retryAfter time.Duration - for i := range allInstances { - stored := &allInstances[i].StoredMetadata - if stored.GPUDevicePath == "" { - continue + m.ReconcileVGPUs(ctx) + m.vgpuReconcileOnce.Do(func() { + interval := m.vgpuReconcileInterval + if interval <= 0 { + interval = defaultVGPUReconcileInterval } - livePID := stored.HypervisorPID != nil && hypervisorMayBeAlive(stored.HypervisorProcessIdentity, stored.SocketPath) - live, remaining := vgpuAssignmentLiveness(stored, m.nowUTC(), livePID) - if !live { - continue - } - protected[stored.GPUDevicePath] = struct{}{} - if remaining > 0 && (retryAfter == 0 || remaining < retryAfter) { - retryAfter = remaining - } - } - return protected, retryAfter, nil + go func() { + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + m.ReconcileVGPUs(ctx) + } + } + }() + }) } -// ReconcileVGPUs releases orphaned vGPU assignments. +// ReconcileVGPUs runs one fail-closed reconcile pass: stale instance-held +// assignments are released, then device-level leftovers not claimed by a live +// instance are swept. Failures only log; the next periodic pass retries. func (m *manager) ReconcileVGPUs(ctx context.Context) { log := logger.FromContext(ctx) - protected, retryAfter, err := m.liveVGPUReconcileProtection(ctx) + protected, err := m.reconcileVGPUAssignments(ctx) sweepVendorVFIO := err == nil if err != nil { - log.ErrorContext(ctx, "failed to list instances for vGPU reconcile protection; skipping vendor VFIO reconcile, mdev reconcile still runs", "error", err) + log.ErrorContext(ctx, "failed to list instances for vGPU reconcile protection; skipping vendor VFIO sweep until the next pass, mdev reconcile still runs", "error", err) protected = make(map[string]struct{}) - retryAfter = vgpuReconcileFailureRetryDelay - if m.vgpuReconcileRetryDelay > 0 { - retryAfter = m.vgpuReconcileRetryDelay - } } reconcileDevices := m.reconcileVGPUDevices if reconcileDevices == nil { @@ -54,29 +65,68 @@ func (m *manager) ReconcileVGPUs(ctx context.Context) { } if err := reconcileDevices(ctx, protected, sweepVendorVFIO); err != nil { log.WarnContext(ctx, "failed to reconcile vGPU devices", "error", err) - deviceRetryAfter := vgpuReconcileFailureRetryDelay - if m.vgpuReconcileRetryDelay > 0 { - deviceRetryAfter = m.vgpuReconcileRetryDelay + } +} + +// reconcileVGPUAssignments retries releases for assignments whose owner is no +// longer live and returns the device paths still protected by live instances. +// Listing fails closed: any unreadable metadata aborts the pass so the vendor +// VFIO sweep cannot clear a VF whose claim it could not read. +func (m *manager) reconcileVGPUAssignments(ctx context.Context) (map[string]struct{}, error) { + allInstances, err := m.listInstancesForReconcile(ctx) + if err != nil { + return nil, err + } + protected := make(map[string]struct{}) + for i := range allInstances { + stored := &allInstances[i].StoredMetadata + if storedVGPUDevicePath(stored) == "" { + continue } - if retryAfter <= 0 || deviceRetryAfter < retryAfter { - retryAfter = deviceRetryAfter + livePID := stored.HypervisorPID != nil && hypervisorMayBeAlive(stored.HypervisorProcessIdentity, stored.SocketPath) + if live, _ := vgpuAssignmentLiveness(stored, m.nowUTC(), livePID); live { + if stored.GPUDevicePath != "" { + protected[stored.GPUDevicePath] = struct{}{} + } + continue } + m.releaseStaleVGPUAssignment(ctx, stored.Id) } - if retryAfter <= 0 { + return protected, nil +} + +// releaseStaleVGPUAssignment retries a release that previously failed, under +// the instance lock. Liveness is re-verified after locking so a concurrent +// start or restore keeps its assignment. A failed release only logs and keeps +// the metadata for the next pass. +func (m *manager) releaseStaleVGPUAssignment(ctx context.Context, id string) { + lock := m.getInstanceLock(id) + lock.Lock() + defer lock.Unlock() + log := logger.FromContext(ctx) + meta, err := m.loadMetadata(id) + if err != nil { + if !errors.Is(err, ErrNotFound) { + log.WarnContext(ctx, "failed to load metadata for stale vGPU release", "instance_id", id, "error", err) + } return } - if !m.vgpuReconcileRetryPending.CompareAndSwap(false, true) { + stored := &meta.StoredMetadata + path := storedVGPUDevicePath(stored) + if path == "" { return } - go func() { - timer := time.NewTimer(retryAfter) - defer timer.Stop() - select { - case <-ctx.Done(): - m.vgpuReconcileRetryPending.Store(false) - case <-timer.C: - m.vgpuReconcileRetryPending.Store(false) - m.ReconcileVGPUs(ctx) - } - }() + livePID := stored.HypervisorPID != nil && hypervisorMayBeAlive(stored.HypervisorProcessIdentity, stored.SocketPath) + if live, _ := vgpuAssignmentLiveness(stored, m.nowUTC(), livePID); live { + return + } + if err := m.releaseStoredVGPU(ctx, stored); err != nil { + log.WarnContext(ctx, "failed to release stale vGPU assignment; retrying on the next reconcile pass", "instance_id", id, "device_path", path, "error", err) + return + } + if err := m.saveMetadata(meta); err != nil { + log.WarnContext(ctx, "failed to save metadata after stale vGPU release", "instance_id", id, "error", err) + return + } + log.InfoContext(ctx, "released stale vGPU assignment", "instance_id", id, "device_path", path) } diff --git a/lib/instances/vgpu_reconcile_test.go b/lib/instances/vgpu_reconcile_test.go index 03851f40d..468591dee 100644 --- a/lib/instances/vgpu_reconcile_test.go +++ b/lib/instances/vgpu_reconcile_test.go @@ -10,12 +10,13 @@ import ( "testing" "time" + "github.com/kernel/hypeman/lib/devices" "github.com/kernel/hypeman/lib/paths" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -func TestLiveVGPUReconcileProtectionBoundsStartupProtection(t *testing.T) { +func TestReconcileVGPUsBoundsStartupProtection(t *testing.T) { dead := exec.Command("true") require.NoError(t, dead.Run()) deadPID := dead.Process.Pid @@ -23,7 +24,17 @@ func TestLiveVGPUReconcileProtectionBoundsStartupProtection(t *testing.T) { recent := now.Add(-time.Minute) stale := now.Add(-VGPUAssignmentStartupGracePeriod - time.Minute) - m := &manager{paths: paths.New(t.TempDir()), now: func() time.Time { return now }} + var protected map[string]struct{} + m := &manager{ + paths: paths.New(t.TempDir()), + now: func() time.Time { return now }, + destroyVGPU: func(context.Context, devices.VGPUAssignment) error { return nil }, + reconcileVGPUDevices: func(_ context.Context, p map[string]struct{}, sweepVendorVFIO bool) error { + protected = p + assert.True(t, sweepVendorVFIO) + return nil + }, + } instances := []StoredMetadata{ {Id: "booting", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", GPUAssignedAt: &recent}, {Id: "orphaned", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.5", GPUAssignedAt: &stale}, @@ -36,22 +47,38 @@ func TestLiveVGPUReconcileProtectionBoundsStartupProtection(t *testing.T) { require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: instances[i]})) } - protected, retryAfter, err := m.liveVGPUReconcileProtection(t.Context()) - require.NoError(t, err) - assert.Equal(t, VGPUAssignmentStartupGracePeriod-time.Minute, retryAfter) + m.ReconcileVGPUs(t.Context()) assert.Contains(t, protected, "/sys/bus/pci/devices/0000:82:00.4") assert.NotContains(t, protected, "/sys/bus/pci/devices/0000:82:00.5") assert.NotContains(t, protected, "/sys/bus/pci/devices/0000:82:00.6") assert.NotContains(t, protected, "/sys/bus/pci/devices/0000:82:00.7") assert.Contains(t, protected, "/sys/bus/pci/devices/0000:82:00.8") + + for _, id := range []string{"orphaned", "legacy", "dead"} { + stored, err := m.loadMetadata(id) + require.NoError(t, err) + assert.Empty(t, stored.GPUDevicePath, "stale assignment on %s must be released", id) + } + for _, id := range []string{"booting", "stale-pid-booting"} { + stored, err := m.loadMetadata(id) + require.NoError(t, err) + assert.NotEmpty(t, stored.GPUDevicePath, "live assignment on %s must be kept", id) + } } -func TestReconcileVGPUsRetriesAfterListingFailure(t *testing.T) { +func TestReconcileVGPUsSkipsVendorSweepWhenListingFails(t *testing.T) { if os.Geteuid() == 0 { t.Skip("root bypasses directory permissions") } - m := &manager{paths: paths.New(t.TempDir()), vgpuReconcileRetryDelay: 250 * time.Millisecond} + var sweeps []bool + m := &manager{ + paths: paths.New(t.TempDir()), + reconcileVGPUDevices: func(_ context.Context, _ map[string]struct{}, sweepVendorVFIO bool) error { + sweeps = append(sweeps, sweepVendorVFIO) + return nil + }, + } const id = "unreadable" require.NoError(t, m.ensureDirectories(id)) require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{Id: id}})) @@ -59,36 +86,181 @@ func TestReconcileVGPUsRetriesAfterListingFailure(t *testing.T) { require.NoError(t, os.Chmod(instanceDir, 0o000)) t.Cleanup(func() { _ = os.Chmod(instanceDir, 0o755) }) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - m.ReconcileVGPUs(ctx) - require.True(t, m.vgpuReconcileRetryPending.Load(), - "a listing failure must schedule a retry instead of disabling the vendor sweep until restart") + m.ReconcileVGPUs(t.Context()) + require.Equal(t, []bool{false}, sweeps, + "a listing failure must skip the vendor sweep, not run it with an empty protection set") require.NoError(t, os.Chmod(instanceDir, 0o755)) - require.Eventually(t, func() bool { - return !m.vgpuReconcileRetryPending.Load() - }, 5*time.Second, 10*time.Millisecond) + m.ReconcileVGPUs(t.Context()) + assert.Equal(t, []bool{false, true}, sweeps, "the next pass retries the vendor sweep") +} + +func TestReconcileVGPUsReleasesStaleAssignment(t *testing.T) { + t.Parallel() + + var destroyed []devices.VGPUAssignment + m := &manager{ + paths: paths.New(t.TempDir()), + destroyVGPU: func(_ context.Context, assignment devices.VGPUAssignment) error { + destroyed = append(destroyed, assignment) + return nil + }, + reconcileVGPUDevices: func(context.Context, map[string]struct{}, bool) error { return nil }, + } + const id = "stopped-retained" + require.NoError(t, m.ensureDirectories(id)) + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: id, + GPUProfile: "NVIDIA L40S-2Q", + GPUFramework: devices.VGPUFrameworkVendorVFIO, + GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", + }})) + + m.ReconcileVGPUs(t.Context()) + + require.Len(t, destroyed, 1) + assert.Equal(t, devices.VGPUAssignment{ + Framework: devices.VGPUFrameworkVendorVFIO, + DevicePath: "/sys/bus/pci/devices/0000:82:00.4", + InstanceID: id, + }, destroyed[0]) + stored, err := m.loadMetadata(id) + require.NoError(t, err) + assert.Empty(t, stored.GPUDevicePath) + assert.Equal(t, "NVIDIA L40S-2Q", stored.GPUProfile, "profile is kept for the next start") +} + +func TestReconcileVGPUsKeepsAssignmentWhenReleaseFails(t *testing.T) { + t.Parallel() + + var attempts atomic.Int32 + m := &manager{ + paths: paths.New(t.TempDir()), + destroyVGPU: func(context.Context, devices.VGPUAssignment) error { + if attempts.Add(1) == 1 { + return errors.New("vGPU destroy failed: 0xffffffff") + } + return nil + }, + reconcileVGPUDevices: func(context.Context, map[string]struct{}, bool) error { return nil }, + } + const id = "wedged" + require.NoError(t, m.ensureDirectories(id)) + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: id, + GPUFramework: devices.VGPUFrameworkVendorVFIO, + GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", + }})) + + m.ReconcileVGPUs(t.Context()) + stored, err := m.loadMetadata(id) + require.NoError(t, err) + assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", stored.GPUDevicePath, + "a failed release must keep the assignment metadata for the next pass") + + m.ReconcileVGPUs(t.Context()) + stored, err = m.loadMetadata(id) + require.NoError(t, err) + assert.Empty(t, stored.GPUDevicePath, "the next pass retries the release") + assert.Equal(t, int32(2), attempts.Load()) +} + +func TestReconcileVGPUsDefersToUnconfirmedClaimant(t *testing.T) { + t.Parallel() + + var destroys atomic.Int32 + m := &manager{ + paths: paths.New(t.TempDir()), + destroyVGPU: func(context.Context, devices.VGPUAssignment) error { + destroys.Add(1) + return nil + }, + reconcileVGPUDevices: func(context.Context, map[string]struct{}, bool) error { return nil }, + } + const path = "/sys/bus/pci/devices/0000:82:00.4" + assignedAt := time.Now() + for _, stored := range []StoredMetadata{ + {Id: "stale", GPUFramework: devices.VGPUFrameworkVendorVFIO, GPUDevicePath: path}, + {Id: "mid-boot-claimant", GPUFramework: devices.VGPUFrameworkVendorVFIO, GPUDevicePath: path, GPUAssignedAt: &assignedAt}, + } { + require.NoError(t, m.ensureDirectories(stored.Id)) + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: stored})) + } + + m.ReconcileVGPUs(t.Context()) + + assert.Zero(t, destroys.Load(), "no destroy may fire while the claim scan cannot clear the path") + stale, err := m.loadMetadata("stale") + require.NoError(t, err) + assert.Equal(t, path, stale.GPUDevicePath, "the stale release retries once the claimant's liveness is decidable") +} + +func TestReconcileVGPUsReleasesRetentionStubAssignment(t *testing.T) { + t.Parallel() + + m := &manager{ + paths: paths.New(t.TempDir()), + destroyVGPU: func(context.Context, devices.VGPUAssignment) error { return nil }, + reconcileVGPUDevices: func(context.Context, map[string]struct{}, bool) error { return nil }, + } + const id = "retention-stub" + require.NoError(t, m.ensureDirectories(id)) + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: id, + Name: "failed-create", + GPUFramework: devices.VGPUFrameworkVendorVFIO, + GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", + GPURetainedForCleanup: true, + }})) + + m.ReconcileVGPUs(t.Context()) + + stored, err := m.loadMetadata(id) + require.NoError(t, err) + assert.Empty(t, stored.GPUDevicePath, "the stub's wedged assignment is released once free") + assert.True(t, stored.GPURetainedForCleanup, "the stub stays a delete-only record of the failed create") } -func TestReconcileVGPUsRetriesAfterDeviceFailure(t *testing.T) { - var calls atomic.Int32 +func TestStartVGPUReconcilerSkipsHostsWithoutGPUs(t *testing.T) { + t.Parallel() + + var passes atomic.Int32 m := &manager{ - paths: paths.New(t.TempDir()), - vgpuReconcileRetryDelay: 10 * time.Millisecond, + paths: paths.New(t.TempDir()), + discoverVGPU: func() (devices.VGPUFramework, []devices.VirtualFunction, error) { + return devices.VGPUFrameworkNone, nil, nil + }, + reconcileVGPUDevices: func(context.Context, map[string]struct{}, bool) error { + passes.Add(1) + return nil + }, + vgpuReconcileInterval: time.Millisecond, + } + + m.StartVGPUReconciler(t.Context()) + time.Sleep(20 * time.Millisecond) + assert.Zero(t, passes.Load(), "a host without GPUs must not reconcile at all") +} + +func TestStartVGPUReconcilerRunsPeriodically(t *testing.T) { + t.Parallel() + + var passes atomic.Int32 + m := &manager{ + paths: paths.New(t.TempDir()), + discoverVGPU: func() (devices.VGPUFramework, []devices.VirtualFunction, error) { + return devices.VGPUFrameworkVendorVFIO, nil, nil + }, reconcileVGPUDevices: func(context.Context, map[string]struct{}, bool) error { - if calls.Add(1) == 1 { + if passes.Add(1) == 1 { return errors.New("transient device error") } return nil }, + vgpuReconcileInterval: time.Millisecond, } - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - m.ReconcileVGPUs(ctx) - require.True(t, m.vgpuReconcileRetryPending.Load()) - require.Eventually(t, func() bool { - return calls.Load() >= 2 && !m.vgpuReconcileRetryPending.Load() - }, 5*time.Second, 10*time.Millisecond) + m.StartVGPUReconciler(t.Context()) + require.Eventually(t, func() bool { return passes.Load() >= 3 }, 5*time.Second, time.Millisecond, + "periodic passes must keep running after a failed pass") } diff --git a/lib/instances/vgpu_retention.go b/lib/instances/vgpu_retention.go index feac72ed2..121c13cb0 100644 --- a/lib/instances/vgpu_retention.go +++ b/lib/instances/vgpu_retention.go @@ -57,6 +57,8 @@ func (m *manager) persistVGPURetention(ctx context.Context, retention *vgpuReten id := retention.instanceID retainedVGPU := retention.stub log := logger.FromContext(ctx) + // When the retention record is lost, the assignment has no metadata claim + // left; the periodic vGPU reconcile sweeps the device once it is free. retentionSurvives := func() bool { meta, err := m.loadMetadata(id) if err == nil && storedVGPUDevicePath(&meta.StoredMetadata) != "" { @@ -65,7 +67,6 @@ func (m *manager) persistVGPURetention(ctx context.Context, retention *vgpuReten if err := m.deleteInstanceData(id); err != nil { log.ErrorContext(ctx, "failed to delete stale instance data after retention failure", "instance_id", id, "error", err) } - m.scheduleOrphanedVGPURelease(ctx, *retainedVGPU) return false } if err := m.deleteInstanceData(id); err != nil { diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index 4b31bf7c7..e5ed8d13f 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -106,12 +106,7 @@ func TestCleanupFailedCreateReportsUnpersistedRetention(t *testing.T) { } assert.False(t, persistTestVGPURetention(m, context.Background(), stored.Id, stored)) _, err := m.loadMetadata(id) - require.Error(t, err) - - m.orphanedVGPUMu.Lock() - _, queued := m.orphanedVGPUs[stored.GPUDevicePath] - m.orphanedVGPUMu.Unlock() - assert.True(t, queued, "unpersisted retention must queue a background release") + require.Error(t, err, "the lost retention leaves no metadata claim, so the periodic sweep releases the VF") } func TestCleanupFailedCreateReportsRetainedWhenFullMetadataSurvives(t *testing.T) { @@ -332,11 +327,6 @@ func TestStartReportsUnretainedVGPUWhenRetentionSaveFails(t *testing.T) { stored, err := m.loadMetadata(id) require.NoError(t, err) assert.Empty(t, stored.GPUDevicePath, "retention save failed, so no assignment should be recorded") - - m.orphanedVGPUMu.Lock() - _, queued := m.orphanedVGPUs[device.SysfsPath] - m.orphanedVGPUMu.Unlock() - assert.True(t, queued, "unpersisted retention must queue a background release") } func TestStartDoesNotRestrictVGPUHypervisor(t *testing.T) { From d1ac2f2742e964232184221cd672194dd0ab3d9d Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Tue, 25 Aug 2026 21:21:01 +0000 Subject: [PATCH 66/76] Update vgpu_cleanup_pending assertions for the periodic reconcile message --- cmd/api/api/instances_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/api/api/instances_test.go b/cmd/api/api/instances_test.go index fd506023d..5f826dc9c 100644 --- a/cmd/api/api/instances_test.go +++ b/cmd/api/api/instances_test.go @@ -104,7 +104,7 @@ func TestCreateInstance_VGPUCleanupPendingWithoutRetentionUsesReconcileGuidance( assert.Contains(t, pending.Message, "retention record for instance inst-1 could not be saved") assert.Contains(t, pending.Message, network.ErrNameExists.Error(), "the underlying create failure must survive the cleanup guidance") - assert.Contains(t, pending.Message, "startup reconcile") + assert.Contains(t, pending.Message, "periodic vGPU reconcile") assert.NotContains(t, pending.Message, "delete") require.NotNil(t, pending.InnerError) require.NotNil(t, pending.InnerError.Code) @@ -1129,7 +1129,7 @@ func TestStartInstance_VGPUCleanupPendingBeatsWrappedErrorMapping(t *testing.T) require.True(t, ok, "expected 500 vgpu_cleanup_pending, got %T", resp) assert.EqualValues(t, "vgpu_cleanup_pending", pending.Code) assert.Contains(t, pending.Message, "retention record for instance inst-1 could not be saved") - assert.Contains(t, pending.Message, "startup reconcile") + assert.Contains(t, pending.Message, "periodic vGPU reconcile") assert.NotContains(t, pending.Message, "delete") require.NotNil(t, pending.InnerError) require.NotNil(t, pending.InnerError.Code) From 93cc7550fcb763c14734d9d578ccdf4c77248b44 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Tue, 25 Aug 2026 21:21:01 +0000 Subject: [PATCH 67/76] Check socket ownership for vGPU liveness without a persisted PID An assignment could lose its persisted hypervisor PID while its VM stays alive - a post-boot metadata save failure or a hypeman crash before the save. The liveness checks gated the socket-ownership scan on a non-nil PID, so once the startup grace expired the reconciler considered such an assignment stale and could remove an mdev out from under the live VM (DestroyMdev has no in-use guard). Run the socket-ownership scan unconditionally: a live VMM always holds its control-socket listener, and a missing listener still resolves to not-alive, so genuinely stopped instances are released as before. The claim scan in releaseStoredVGPUExcluding gets the same treatment. --- lib/instances/vgpu.go | 15 +++--- lib/instances/vgpu_reconcile.go | 8 ++- lib/instances/vgpu_reconcile_linux_test.go | 62 ++++++++++++++++++++++ 3 files changed, 75 insertions(+), 10 deletions(-) create mode 100644 lib/instances/vgpu_reconcile_linux_test.go diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index 3fb26778a..bb230f3d1 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -13,7 +13,7 @@ import ( // VGPUAssignmentStartupGracePeriod bounds how long an assignment without a // persisted hypervisor PID is treated as potentially live. -const VGPUAssignmentStartupGracePeriod = 5 * time.Minute +const VGPUAssignmentStartupGracePeriod = devices.VGPUAssignmentGracePeriod // VGPUCleanupPendingError reports a failed rollback that left a vGPU assigned. type VGPUCleanupPendingError struct { @@ -50,7 +50,7 @@ func vgpuDevicePendingCleanup(err error) (*devices.VGPUDevice, bool) { } func vgpuAssignmentLiveness(stored *StoredMetadata, now time.Time, livePID bool) (live bool, graceRemaining time.Duration) { - if stored.HypervisorPID != nil && livePID { + if livePID { return true, 0 } if stored.GPUAssignedAt == nil { @@ -198,12 +198,11 @@ func (m *manager) vgpuAssignmentClaimedByLiveInstance(ctx context.Context, exclu if storedVGPUDevicePath(stored) != devicePath { continue } - pid := 0 - if stored.HypervisorPID != nil { - pid, err = resolveLiveHypervisorPID(stored.HypervisorProcessIdentity, stored.SocketPath) - if err != nil { - return false, fmt.Errorf("cannot confirm liveness of vGPU claimant %s on %s: %w", id, devicePath, err) - } + // Resolve even without a persisted PID: the socket-ownership scan can + // still prove a claimant whose post-boot metadata save failed is live. + pid, err := resolveLiveHypervisorPID(stored.HypervisorProcessIdentity, stored.SocketPath) + if err != nil { + return false, fmt.Errorf("cannot confirm liveness of vGPU claimant %s on %s: %w", id, devicePath, err) } live, remaining := vgpuAssignmentLiveness(stored, m.nowUTC(), pid > 0) if pid > 0 && live { diff --git a/lib/instances/vgpu_reconcile.go b/lib/instances/vgpu_reconcile.go index 966bf1118..5cfb43d4c 100644 --- a/lib/instances/vgpu_reconcile.go +++ b/lib/instances/vgpu_reconcile.go @@ -83,7 +83,11 @@ func (m *manager) reconcileVGPUAssignments(ctx context.Context) (map[string]stru if storedVGPUDevicePath(stored) == "" { continue } - livePID := stored.HypervisorPID != nil && hypervisorMayBeAlive(stored.HypervisorProcessIdentity, stored.SocketPath) + // The socket-ownership check runs even without a persisted PID: a VMM + // whose post-boot metadata save failed still holds its control-socket + // listener, and releasing its device would tear the vGPU out from + // under a live VM. + livePID := hypervisorMayBeAlive(stored.HypervisorProcessIdentity, stored.SocketPath) if live, _ := vgpuAssignmentLiveness(stored, m.nowUTC(), livePID); live { if stored.GPUDevicePath != "" { protected[stored.GPUDevicePath] = struct{}{} @@ -116,7 +120,7 @@ func (m *manager) releaseStaleVGPUAssignment(ctx context.Context, id string) { if path == "" { return } - livePID := stored.HypervisorPID != nil && hypervisorMayBeAlive(stored.HypervisorProcessIdentity, stored.SocketPath) + livePID := hypervisorMayBeAlive(stored.HypervisorProcessIdentity, stored.SocketPath) if live, _ := vgpuAssignmentLiveness(stored, m.nowUTC(), livePID); live { return } diff --git a/lib/instances/vgpu_reconcile_linux_test.go b/lib/instances/vgpu_reconcile_linux_test.go new file mode 100644 index 000000000..8708a1dd7 --- /dev/null +++ b/lib/instances/vgpu_reconcile_linux_test.go @@ -0,0 +1,62 @@ +//go:build linux + +package instances + +import ( + "context" + "net" + "path/filepath" + "testing" + "time" + + "github.com/kernel/hypeman/lib/devices" + "github.com/kernel/hypeman/lib/paths" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// A VMM whose post-boot metadata save failed has no persisted PID, but it +// still holds its control-socket listener. The reconciler must protect its +// assignment past the startup grace period instead of releasing the device +// out from under the live VM. +func TestReconcileVGPUsProtectsSocketOwnerWithoutPersistedPID(t *testing.T) { + t.Parallel() + + socketPath := filepath.Join(t.TempDir(), "test.sock") + listener, err := net.Listen("unix", socketPath) + require.NoError(t, err) + defer listener.Close() + + var destroyed []devices.VGPUAssignment + var protected map[string]struct{} + m := &manager{ + paths: paths.New(t.TempDir()), + destroyVGPU: func(_ context.Context, assignment devices.VGPUAssignment) error { + destroyed = append(destroyed, assignment) + return nil + }, + reconcileVGPUDevices: func(_ context.Context, p map[string]struct{}, _ bool) error { + protected = p + return nil + }, + } + const id = "pid-save-failed" + stale := time.Now().UTC().Add(-VGPUAssignmentStartupGracePeriod - time.Minute) + require.NoError(t, m.ensureDirectories(id)) + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: id, + SocketPath: socketPath, + GPUFramework: devices.VGPUFrameworkMdev, + GPUDevicePath: "/sys/bus/mdev/devices/test-mdev", + GPUMdevUUID: "test-mdev", + GPUAssignedAt: &stale, + }})) + + m.ReconcileVGPUs(t.Context()) + + assert.Empty(t, destroyed, "a live socket owner must block the release even with a nil persisted PID") + assert.Contains(t, protected, "/sys/bus/mdev/devices/test-mdev") + stored, err := m.loadMetadata(id) + require.NoError(t, err) + assert.Equal(t, "/sys/bus/mdev/devices/test-mdev", stored.GPUDevicePath) +} From a7886fe6c2ac1d026c8a76f8f4d3b60680e1f5a0 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Tue, 25 Aug 2026 21:21:01 +0000 Subject: [PATCH 68/76] Share one vGPU assignment grace period constant The five-minute fresh-assignment protection existed three times: the instances startup grace, the mdev orphan grace, and the vendor VFIO sweep grace. Define it once in lib/devices and alias the instances constant to it. --- lib/devices/mdev_linux.go | 13 ++++++------- lib/devices/types.go | 5 +++++ lib/devices/vendor_vfio_linux.go | 7 +------ lib/devices/vendor_vfio_linux_test.go | 2 +- 4 files changed, 13 insertions(+), 14 deletions(-) diff --git a/lib/devices/mdev_linux.go b/lib/devices/mdev_linux.go index 6ef9f6c60..0908e8765 100644 --- a/lib/devices/mdev_linux.go +++ b/lib/devices/mdev_linux.go @@ -23,10 +23,9 @@ import ( ) const ( - mdevBusPath = "/sys/class/mdev_bus" - mdevDevices = "/sys/bus/mdev/devices" - orphanedMdevGracePeriod = 5 * time.Minute - procPath = "/proc" + mdevBusPath = "/sys/class/mdev_bus" + mdevDevices = "/sys/bus/mdev/devices" + procPath = "/proc" ) // mdevMu protects mdev creation/destruction to prevent race conditions @@ -747,7 +746,7 @@ func ReconcileMdevs(ctx context.Context, instanceInfos []MdevReconcileInfo) erro log.InfoContext(ctx, "reconciling mdev devices", "total_mdevs", len(mdevs), "managed_vfs", len(managedVFs), - "grace_period", orphanedMdevGracePeriod.String(), + "grace_period", VGPUAssignmentGracePeriod.String(), ) groupInUseCache := make(map[int]bool) @@ -782,7 +781,7 @@ func ReconcileMdevs(ctx context.Context, instanceInfos []MdevReconcileInfo) erro continue } - pastGracePeriod, age, err := mdevPastGracePeriod(mdev.UUID, orphanedMdevGracePeriod) + pastGracePeriod, age, err := mdevPastGracePeriod(mdev.UUID, VGPUAssignmentGracePeriod) if err != nil { log.WarnContext(ctx, "failed to determine mdev age, skipping cleanup", "uuid", mdev.UUID, "error", err) skippedProbeError++ @@ -792,7 +791,7 @@ func ReconcileMdevs(ctx context.Context, instanceInfos []MdevReconcileInfo) erro log.DebugContext(ctx, "skipping recently created mdev during grace period", "uuid", mdev.UUID, "age", age.String(), - "grace_period", orphanedMdevGracePeriod.String(), + "grace_period", VGPUAssignmentGracePeriod.String(), ) skippedGrace++ continue diff --git a/lib/devices/types.go b/lib/devices/types.go index 31ebd90ef..369bb3268 100644 --- a/lib/devices/types.go +++ b/lib/devices/types.go @@ -60,6 +60,11 @@ func ValidateDeviceName(name string) bool { // GPUMode represents the host's GPU configuration mode type GPUMode string +// VGPUAssignmentGracePeriod protects a fresh vGPU assignment from cleanup +// until its VM has had time to boot and become identifiable — by a persisted +// hypervisor PID, a control-socket owner, or an open VFIO handle. +const VGPUAssignmentGracePeriod = 5 * time.Minute + type VGPUFramework string const ( diff --git a/lib/devices/vendor_vfio_linux.go b/lib/devices/vendor_vfio_linux.go index 74eae4948..a1378a3c7 100644 --- a/lib/devices/vendor_vfio_linux.go +++ b/lib/devices/vendor_vfio_linux.go @@ -23,11 +23,6 @@ import ( const ( pciDevicesPath = "/sys/bus/pci/devices" vfioDevicesPath = "/dev/vfio/devices" - - // vendorVFIOAssignmentGracePeriod protects assignments created by this - // process from the periodic sweep until their owning instance has had time - // to persist metadata and boot, mirroring orphanedMdevGracePeriod. - vendorVFIOAssignmentGracePeriod = 5 * time.Minute ) type vendorVFIOOwner struct { @@ -294,7 +289,7 @@ func (s vendorVFIOSysfs) reconcile(ctx context.Context, protectedDevicePaths map continue } owner := owners[vf.PCIAddress] - if !owner.assignedAt.IsZero() && time.Since(owner.assignedAt) < vendorVFIOAssignmentGracePeriod { + if !owner.assignedAt.IsZero() && time.Since(owner.assignedAt) < VGPUAssignmentGracePeriod { log.DebugContext(ctx, "skipping recently assigned vendor VFIO vGPU during grace period", "vf", vf.PCIAddress) continue } diff --git a/lib/devices/vendor_vfio_linux_test.go b/lib/devices/vendor_vfio_linux_test.go index f7b0d6c3f..098d016e3 100644 --- a/lib/devices/vendor_vfio_linux_test.go +++ b/lib/devices/vendor_vfio_linux_test.go @@ -391,7 +391,7 @@ func TestVendorVFIOReconcileDestroysOwnedVFPastGracePeriod(t *testing.T) { sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "1148", "") sysfs.owners["0000:82:00.4"] = vendorVFIOOwner{ instanceID: "deleted-instance", - assignedAt: time.Now().Add(-vendorVFIOAssignmentGracePeriod - time.Minute), + assignedAt: time.Now().Add(-VGPUAssignmentGracePeriod - time.Minute), } require.NoError(t, sysfs.reconcile(context.Background(), nil)) From 34f606a17c6a8b1ce226e15309f29ac57cfa0924 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Wed, 26 Aug 2026 15:09:21 +0000 Subject: [PATCH 69/76] Preserve delete-only vGPU retention records --- lib/instances/vgpu_retention.go | 81 +++++++++++++++++++++++++-------- lib/instances/vgpu_test.go | 6 ++- 2 files changed, 66 insertions(+), 21 deletions(-) diff --git a/lib/instances/vgpu_retention.go b/lib/instances/vgpu_retention.go index 121c13cb0..e630484ae 100644 --- a/lib/instances/vgpu_retention.go +++ b/lib/instances/vgpu_retention.go @@ -2,9 +2,13 @@ package instances import ( "context" + "encoding/json" + "fmt" + "os" "time" "github.com/kernel/hypeman/lib/devices" + "github.com/kernel/hypeman/lib/hypervisor" "github.com/kernel/hypeman/lib/logger" ) @@ -62,7 +66,16 @@ func (m *manager) persistVGPURetention(ctx context.Context, retention *vgpuReten retentionSurvives := func() bool { meta, err := m.loadMetadata(id) if err == nil && storedVGPUDevicePath(&meta.StoredMetadata) != "" { - return true + saveErr := m.saveVGPURetentionStub(retainedVGPU) + if saveErr == nil { + return true + } + log.ErrorContext(ctx, "failed to replace surviving instance metadata with vGPU retention stub", "instance_id", id, "error", saveErr) + overwriteErr := m.overwriteVGPURetentionStub(retainedVGPU) + if overwriteErr == nil { + return true + } + log.ErrorContext(ctx, "failed to overwrite surviving instance metadata with vGPU retention stub", "instance_id", id, "error", overwriteErr) } if err := m.deleteInstanceData(id); err != nil { log.ErrorContext(ctx, "failed to delete stale instance data after retention failure", "instance_id", id, "error", err) @@ -79,28 +92,56 @@ func (m *manager) persistVGPURetention(ctx context.Context, retention *vgpuReten retention.persisted = retentionSurvives() return } - retained := StoredMetadata{ - Id: id, - Name: retainedVGPU.Name, - Image: retainedVGPU.Image, - ResolvedImage: retainedVGPU.ResolvedImage, - Platform: retainedVGPU.Platform, - CreatedAt: retainedVGPU.CreatedAt, - HypervisorType: retainedVGPU.HypervisorType, - HypervisorVersion: retainedVGPU.HypervisorVersion, - SocketPath: retainedVGPU.SocketPath, - DataDir: retainedVGPU.DataDir, - GPUProfile: retainedVGPU.GPUProfile, - GPUFramework: retainedVGPU.GPUFramework, - GPUDevicePath: retainedVGPU.GPUDevicePath, - GPUMdevUUID: retainedVGPU.GPUMdevUUID, - GPUAssignedAt: retainedVGPU.GPUAssignedAt, - GPURetainedForCleanup: true, - } - if err := m.saveMetadata(&metadata{StoredMetadata: retained}); err != nil { + if err := m.saveVGPURetentionStub(retainedVGPU); err != nil { log.ErrorContext(ctx, "failed to retain vGPU assignment metadata after cleanup failure", "instance_id", id, "error", err) retention.persisted = retentionSurvives() return } retention.persisted = true } + +func vgpuRetentionMetadata(source *StoredMetadata) *metadata { + return &metadata{StoredMetadata: StoredMetadata{ + Id: source.Id, + Name: source.Name, + Image: source.Image, + ResolvedImage: source.ResolvedImage, + Platform: source.Platform, + CreatedAt: source.CreatedAt, + HypervisorType: source.HypervisorType, + HypervisorVersion: source.HypervisorVersion, + SocketPath: source.SocketPath, + DataDir: source.DataDir, + GPUProfile: source.GPUProfile, + GPUFramework: source.GPUFramework, + GPUDevicePath: source.GPUDevicePath, + GPUMdevUUID: source.GPUMdevUUID, + GPUAssignedAt: source.GPUAssignedAt, + GPURetainedForCleanup: true, + }} +} + +func (m *manager) saveVGPURetentionStub(source *StoredMetadata) error { + return m.saveMetadata(vgpuRetentionMetadata(source)) +} + +// overwriteVGPURetentionStub handles a surviving metadata file when its +// directory cannot create the temporary file used by saveMetadata. +func (m *manager) overwriteVGPURetentionStub(source *StoredMetadata) error { + retained := vgpuRetentionMetadata(source) + data, err := json.MarshalIndent(retained, "", " ") + if err != nil { + return fmt.Errorf("marshal metadata: %w", err) + } + unlockAliasReaders := hypervisor.LockSnapshotSourceAliasReaders() + defer unlockAliasReaders() + writeFile := m.writeFile + if writeFile == nil { + writeFile = os.WriteFile + } + if err := writeFile(m.paths.InstanceMetadata(source.Id), data, 0644); err != nil { + return fmt.Errorf("write metadata: %w", err) + } + m.syncAdmissionAllocation(retained) + return nil +} diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index e5ed8d13f..b9f7124d2 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -109,7 +109,7 @@ func TestCleanupFailedCreateReportsUnpersistedRetention(t *testing.T) { require.Error(t, err, "the lost retention leaves no metadata claim, so the periodic sweep releases the VF") } -func TestCleanupFailedCreateReportsRetainedWhenFullMetadataSurvives(t *testing.T) { +func TestCleanupFailedCreateRewritesFullMetadataWhenDirectoryIsReadOnly(t *testing.T) { if os.Geteuid() == 0 { t.Skip("root bypasses directory permissions") } @@ -133,6 +133,10 @@ func TestCleanupFailedCreateReportsRetainedWhenFullMetadataSurvives(t *testing.T require.NoError(t, err) assert.Equal(t, stored.GPUFramework, retained.GPUFramework) assert.Equal(t, stored.GPUDevicePath, retained.GPUDevicePath) + assert.True(t, retained.GPURetainedForCleanup) + + _, err = m.startInstance(context.Background(), id, StartInstanceRequest{}) + assert.ErrorIs(t, err, errVGPURetentionStub) } func TestVGPURetentionWrapPending(t *testing.T) { From 742281ec1ff2b208541450050bd0d53f8a6db31a Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Wed, 26 Aug 2026 15:44:10 +0000 Subject: [PATCH 70/76] Trim redundant vGPU coverage --- cmd/api/api/instances_test.go | 164 +++++++++------------------ lib/instances/lifecycle_noop_test.go | 18 +-- lib/instances/vgpu.go | 10 +- lib/instances/vgpu_reconcile_test.go | 55 +++------ lib/instances/vgpu_test.go | 117 +++++++------------ 5 files changed, 116 insertions(+), 248 deletions(-) diff --git a/cmd/api/api/instances_test.go b/cmd/api/api/instances_test.go index 5f826dc9c..ccaeb3a86 100644 --- a/cmd/api/api/instances_test.go +++ b/cmd/api/api/instances_test.go @@ -48,69 +48,37 @@ func TestGetInstance_NotFound(t *testing.T) { require.Error(t, err) } -type createErrorInstanceManager struct { - instances.Manager - err error -} - -func (m createErrorInstanceManager) CreateInstance(context.Context, instances.CreateInstanceRequest) (*instances.Instance, error) { - return nil, m.err -} - -func TestCreateInstance_VGPUCleanupPendingBeatsWrappedErrorMapping(t *testing.T) { +func TestVGPUCleanupPendingDetail(t *testing.T) { t.Parallel() - svc := newTestService(t) - svc.InstanceManager = createErrorInstanceManager{err: &instances.VGPUCleanupPendingError{ - InstanceID: "inst-1", - Retained: true, - Err: network.ErrNameExists, - }} - - resp, err := svc.CreateInstance(ctx(), oapi.CreateInstanceRequestObject{ - Body: &oapi.CreateInstanceRequest{Image: "test-image"}, - }) - require.NoError(t, err) - - pending, ok := resp.(oapi.CreateInstance500JSONResponse) - require.True(t, ok, "expected 500 vgpu_cleanup_pending, got %T", resp) - assert.EqualValues(t, "vgpu_cleanup_pending", pending.Code) - assert.Contains(t, pending.Message, "inst-1") - assert.Contains(t, pending.Message, network.ErrNameExists.Error(), - "the underlying create failure must survive the cleanup guidance") - assert.Contains(t, pending.Message, "delete it to retry") - require.NotNil(t, pending.InnerError) - require.NotNil(t, pending.InnerError.Code) - assert.Equal(t, "vgpu_retained_instance", *pending.InnerError.Code) - require.NotNil(t, pending.InnerError.Message) - assert.Equal(t, "inst-1", *pending.InnerError.Message) -} - -func TestCreateInstance_VGPUCleanupPendingWithoutRetentionUsesReconcileGuidance(t *testing.T) { - t.Parallel() - svc := newTestService(t) - svc.InstanceManager = createErrorInstanceManager{err: &instances.VGPUCleanupPendingError{ - InstanceID: "inst-1", - Err: network.ErrNameExists, - }} - - resp, err := svc.CreateInstance(ctx(), oapi.CreateInstanceRequestObject{ - Body: &oapi.CreateInstanceRequest{Image: "test-image"}, - }) - require.NoError(t, err) - - pending, ok := resp.(oapi.CreateInstance500JSONResponse) - require.True(t, ok, "expected 500 vgpu_cleanup_pending, got %T", resp) - assert.EqualValues(t, "vgpu_cleanup_pending", pending.Code) - assert.Contains(t, pending.Message, "retention record for instance inst-1 could not be saved") - assert.Contains(t, pending.Message, network.ErrNameExists.Error(), - "the underlying create failure must survive the cleanup guidance") - assert.Contains(t, pending.Message, "periodic vGPU reconcile") - assert.NotContains(t, pending.Message, "delete") - require.NotNil(t, pending.InnerError) - require.NotNil(t, pending.InnerError.Code) - assert.Equal(t, "vgpu_unretained_instance", *pending.InnerError.Code) - require.NotNil(t, pending.InnerError.Message) - assert.Equal(t, "inst-1", *pending.InnerError.Message) + for _, tt := range []struct { + name string + retained bool + code string + guidance string + exclude string + }{ + {name: "retained", retained: true, code: "vgpu_retained_instance", guidance: "delete it to retry"}, + {name: "unretained", code: "vgpu_unretained_instance", guidance: "periodic vGPU reconcile", exclude: "delete"}, + } { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + message, inner := vgpuCleanupPendingDetail(&instances.VGPUCleanupPendingError{ + InstanceID: "inst-1", + Retained: tt.retained, + Err: network.ErrNameExists, + }, "create", "delete it to retry") + assert.Contains(t, message, "inst-1") + assert.Contains(t, message, network.ErrNameExists.Error()) + assert.Contains(t, message, tt.guidance) + if tt.exclude != "" { + assert.NotContains(t, message, tt.exclude) + } + require.NotNil(t, inner.Code) + assert.Equal(t, tt.code, *inner.Code) + require.NotNil(t, inner.Message) + assert.Equal(t, "inst-1", *inner.Message) + }) + } } func TestCreateInstance_AutoPullImage(t *testing.T) { @@ -956,6 +924,16 @@ func TestCreateInstance_ErrorStatusMapping(t *testing.T) { wantCode string wantMessage string }{ + { + name: "vGPU cleanup pending beats wrapped name conflict -> 500", + err: &instances.VGPUCleanupPendingError{ + InstanceID: "inst-1", + Retained: true, + Err: network.ErrNameExists, + }, + wantType: oapi.CreateInstance500JSONResponse{}, + wantCode: "vgpu_cleanup_pending", + }, { name: "platform not available -> 404", err: fmt.Errorf("resolve image: %w", images.ErrPlatformNotAvailable), @@ -1082,61 +1060,23 @@ func TestRestoreInstance_ErrorMapping(t *testing.T) { func TestStartInstance_VGPUCleanupPendingBeatsWrappedErrorMapping(t *testing.T) { t.Parallel() - + svc := newTestService(t) resolved := &instances.Instance{ StoredMetadata: instances.StoredMetadata{Id: "inst-1", Name: "inst-1"}, State: instances.StateStopped, } + svc.InstanceManager = &errActionInstanceManager{Manager: svc.InstanceManager, err: &instances.VGPUCleanupPendingError{ + InstanceID: resolved.Id, + Retained: true, + Err: fmt.Errorf("create vGPU for profile p: %w", instances.ErrInsufficientResources), + }} - t.Run("retained", func(t *testing.T) { - t.Parallel() - svc := newTestService(t) - svc.InstanceManager = &errActionInstanceManager{Manager: svc.InstanceManager, err: &instances.VGPUCleanupPendingError{ - InstanceID: "inst-1", - Retained: true, - Err: fmt.Errorf("create vGPU for profile p: %w", instances.ErrInsufficientResources), - }} - - resp, rerr := svc.StartInstance(mw.WithResolvedInstance(ctx(), resolved.Id, resolved), oapi.StartInstanceRequestObject{Id: resolved.Id}) - require.NoError(t, rerr) - - pending, ok := resp.(oapi.StartInstance500JSONResponse) - require.True(t, ok, "expected 500 vgpu_cleanup_pending, got %T", resp) - assert.EqualValues(t, "vgpu_cleanup_pending", pending.Code) - assert.Contains(t, pending.Message, "inst-1") - assert.Contains(t, pending.Message, instances.ErrInsufficientResources.Error(), - "the underlying start failure must survive the cleanup guidance") - assert.Contains(t, pending.Message, "delete it or retry start") - require.NotNil(t, pending.InnerError) - require.NotNil(t, pending.InnerError.Code) - assert.Equal(t, "vgpu_retained_instance", *pending.InnerError.Code) - require.NotNil(t, pending.InnerError.Message) - assert.Equal(t, "inst-1", *pending.InnerError.Message) - }) - - t.Run("unretained", func(t *testing.T) { - t.Parallel() - svc := newTestService(t) - svc.InstanceManager = &errActionInstanceManager{Manager: svc.InstanceManager, err: &instances.VGPUCleanupPendingError{ - InstanceID: "inst-1", - Err: fmt.Errorf("create vGPU for profile p: %w", instances.ErrInsufficientResources), - }} - - resp, rerr := svc.StartInstance(mw.WithResolvedInstance(ctx(), resolved.Id, resolved), oapi.StartInstanceRequestObject{Id: resolved.Id}) - require.NoError(t, rerr) - - pending, ok := resp.(oapi.StartInstance500JSONResponse) - require.True(t, ok, "expected 500 vgpu_cleanup_pending, got %T", resp) - assert.EqualValues(t, "vgpu_cleanup_pending", pending.Code) - assert.Contains(t, pending.Message, "retention record for instance inst-1 could not be saved") - assert.Contains(t, pending.Message, "periodic vGPU reconcile") - assert.NotContains(t, pending.Message, "delete") - require.NotNil(t, pending.InnerError) - require.NotNil(t, pending.InnerError.Code) - assert.Equal(t, "vgpu_unretained_instance", *pending.InnerError.Code) - require.NotNil(t, pending.InnerError.Message) - assert.Equal(t, "inst-1", *pending.InnerError.Message) - }) + resp, err := svc.StartInstance(mw.WithResolvedInstance(ctx(), resolved.Id, resolved), oapi.StartInstanceRequestObject{Id: resolved.Id}) + require.NoError(t, err) + pending, ok := resp.(oapi.StartInstance500JSONResponse) + require.True(t, ok, "expected 500 vgpu_cleanup_pending, got %T", resp) + assert.EqualValues(t, "vgpu_cleanup_pending", pending.Code) + assert.Contains(t, pending.Message, "delete it or retry start") } func TestInstanceActions_ImageNotFoundMapsTo404(t *testing.T) { diff --git a/lib/instances/lifecycle_noop_test.go b/lib/instances/lifecycle_noop_test.go index 1843485e3..85083766b 100644 --- a/lib/instances/lifecycle_noop_test.go +++ b/lib/instances/lifecycle_noop_test.go @@ -214,10 +214,11 @@ func TestDeleteReleasesRetainedCreateStub(t *testing.T) { require.NoError(t, m.ensureDirectories(id)) assignedAt := time.Now().UTC() require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ - Id: id, - GPUFramework: devices.VGPUFrameworkVendorVFIO, - GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", - GPUAssignedAt: &assignedAt, + Id: id, + GPUFramework: devices.VGPUFrameworkVendorVFIO, + GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", + GPUAssignedAt: &assignedAt, + GPURetainedForCleanup: true, }})) require.NoError(t, m.DeleteInstance(context.Background(), id)) @@ -339,9 +340,8 @@ func TestStartRejectsVGPURetentionRecord(t *testing.T) { assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", stored.GPUDevicePath) } -func TestReconcileReleasesRetainedVGPUOnStoppedInstance(t *testing.T) { +func TestStopStoppedInstanceLeavesVGPUForReconcile(t *testing.T) { m, id := newLifecycleNoopManagerWithInstance(t, StateStopped, time.Now().UTC()) - m.reconcileVGPUDevices = func(context.Context, map[string]struct{}, bool) error { return nil } meta, err := m.loadMetadata(id) require.NoError(t, err) meta.GPUProfile = "NVIDIA L40S-2Q" @@ -358,12 +358,6 @@ func TestReconcileReleasesRetainedVGPUOnStoppedInstance(t *testing.T) { stored, err := m.loadMetadata(id) require.NoError(t, err) assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", stored.GPUDevicePath) - - m.ReconcileVGPUs(context.Background()) - stored, err = m.loadMetadata(id) - require.NoError(t, err) - assert.Empty(t, stored.GPUDevicePath) - assert.Equal(t, "NVIDIA L40S-2Q", stored.GPUProfile, "profile is kept for the next start") } func TestReconcileVGPUReleaseFailureKeepsStoppedInstanceUsable(t *testing.T) { diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index bb230f3d1..37127e921 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -143,17 +143,13 @@ func restoreStartMutatedFields(dst, src *StoredMetadata) { } func (m *manager) releaseStoredVGPU(ctx context.Context, stored *StoredMetadata) error { - return m.releaseStoredVGPUExcluding(ctx, stored, stored.Id) -} - -func (m *manager) releaseStoredVGPUExcluding(ctx context.Context, stored *StoredMetadata, excludeID string) error { path := storedVGPUDevicePath(stored) if path != "" { // Vendor VFIO VFs are reusable, so release fails closed on an incomplete inventory. claimed := false if stored.GPUFramework == devices.VGPUFrameworkVendorVFIO { var err error - claimed, err = m.vgpuAssignmentClaimedByLiveInstance(ctx, excludeID, path) + claimed, err = m.vgpuAssignmentClaimedByLiveInstance(ctx, stored.Id, path) if err != nil { return err } @@ -204,8 +200,8 @@ func (m *manager) vgpuAssignmentClaimedByLiveInstance(ctx context.Context, exclu if err != nil { return false, fmt.Errorf("cannot confirm liveness of vGPU claimant %s on %s: %w", id, devicePath, err) } - live, remaining := vgpuAssignmentLiveness(stored, m.nowUTC(), pid > 0) - if pid > 0 && live { + _, remaining := vgpuAssignmentLiveness(stored, m.nowUTC(), pid > 0) + if pid > 0 { return true, nil } if remaining > 0 { diff --git a/lib/instances/vgpu_reconcile_test.go b/lib/instances/vgpu_reconcile_test.go index 468591dee..395010532 100644 --- a/lib/instances/vgpu_reconcile_test.go +++ b/lib/instances/vgpu_reconcile_test.go @@ -25,10 +25,14 @@ func TestReconcileVGPUsBoundsStartupProtection(t *testing.T) { stale := now.Add(-VGPUAssignmentStartupGracePeriod - time.Minute) var protected map[string]struct{} + var destroyed []devices.VGPUAssignment m := &manager{ - paths: paths.New(t.TempDir()), - now: func() time.Time { return now }, - destroyVGPU: func(context.Context, devices.VGPUAssignment) error { return nil }, + paths: paths.New(t.TempDir()), + now: func() time.Time { return now }, + destroyVGPU: func(_ context.Context, assignment devices.VGPUAssignment) error { + destroyed = append(destroyed, assignment) + return nil + }, reconcileVGPUDevices: func(_ context.Context, p map[string]struct{}, sweepVendorVFIO bool) error { protected = p assert.True(t, sweepVendorVFIO) @@ -37,7 +41,7 @@ func TestReconcileVGPUsBoundsStartupProtection(t *testing.T) { } instances := []StoredMetadata{ {Id: "booting", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", GPUAssignedAt: &recent}, - {Id: "orphaned", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.5", GPUAssignedAt: &stale}, + {Id: "orphaned", GPUProfile: "NVIDIA L40S-2Q", GPUFramework: devices.VGPUFrameworkVendorVFIO, GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.5", GPUAssignedAt: &stale}, {Id: "legacy", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.6"}, {Id: "dead", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.7", HypervisorProcessIdentity: HypervisorProcessIdentity{HypervisorPID: &deadPID}}, {Id: "stale-pid-booting", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.8", HypervisorProcessIdentity: HypervisorProcessIdentity{HypervisorPID: &deadPID}, GPUAssignedAt: &recent}, @@ -59,6 +63,14 @@ func TestReconcileVGPUsBoundsStartupProtection(t *testing.T) { require.NoError(t, err) assert.Empty(t, stored.GPUDevicePath, "stale assignment on %s must be released", id) } + assert.Contains(t, destroyed, devices.VGPUAssignment{ + Framework: devices.VGPUFrameworkVendorVFIO, + DevicePath: "/sys/bus/pci/devices/0000:82:00.5", + InstanceID: "orphaned", + }) + orphaned, err := m.loadMetadata("orphaned") + require.NoError(t, err) + assert.Equal(t, "NVIDIA L40S-2Q", orphaned.GPUProfile) for _, id := range []string{"booting", "stale-pid-booting"} { stored, err := m.loadMetadata(id) require.NoError(t, err) @@ -95,41 +107,6 @@ func TestReconcileVGPUsSkipsVendorSweepWhenListingFails(t *testing.T) { assert.Equal(t, []bool{false, true}, sweeps, "the next pass retries the vendor sweep") } -func TestReconcileVGPUsReleasesStaleAssignment(t *testing.T) { - t.Parallel() - - var destroyed []devices.VGPUAssignment - m := &manager{ - paths: paths.New(t.TempDir()), - destroyVGPU: func(_ context.Context, assignment devices.VGPUAssignment) error { - destroyed = append(destroyed, assignment) - return nil - }, - reconcileVGPUDevices: func(context.Context, map[string]struct{}, bool) error { return nil }, - } - const id = "stopped-retained" - require.NoError(t, m.ensureDirectories(id)) - require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ - Id: id, - GPUProfile: "NVIDIA L40S-2Q", - GPUFramework: devices.VGPUFrameworkVendorVFIO, - GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", - }})) - - m.ReconcileVGPUs(t.Context()) - - require.Len(t, destroyed, 1) - assert.Equal(t, devices.VGPUAssignment{ - Framework: devices.VGPUFrameworkVendorVFIO, - DevicePath: "/sys/bus/pci/devices/0000:82:00.4", - InstanceID: id, - }, destroyed[0]) - stored, err := m.loadMetadata(id) - require.NoError(t, err) - assert.Empty(t, stored.GPUDevicePath) - assert.Equal(t, "NVIDIA L40S-2Q", stored.GPUProfile, "profile is kept for the next start") -} - func TestReconcileVGPUsKeepsAssignmentWhenReleaseFails(t *testing.T) { t.Parallel() diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index b9f7124d2..55f88f4d9 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -526,87 +526,48 @@ func TestVGPUAssignmentClaimedByLiveInstanceNormalizesLegacyMdevPath(t *testing. assert.True(t, claimed) } -func TestVGPUAssignmentClaimedByLiveInstanceErrorsOnRecentNilPIDClaim(t *testing.T) { +func TestVGPUAssignmentClaimedByLiveInstanceLiveness(t *testing.T) { t.Parallel() - m := &manager{paths: paths.New(t.TempDir())} - require.NoError(t, m.ensureDirectories("booting-claimant")) - assignedAt := time.Now().UTC() - require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ - Id: "booting-claimant", - Name: "booting-claimant", - GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", - GPUAssignedAt: &assignedAt, - }})) - - _, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "other-instance", "/sys/bus/pci/devices/0000:82:00.4") - require.Error(t, err) - assert.Contains(t, err.Error(), "booting-claimant") -} - -func TestVGPUAssignmentClaimedByLiveInstanceIgnoresStaleNilPIDClaim(t *testing.T) { - t.Parallel() - - m := &manager{paths: paths.New(t.TempDir())} - require.NoError(t, m.ensureDirectories("stale-claimant")) - assignedAt := time.Now().Add(-VGPUAssignmentStartupGracePeriod - time.Minute) - require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ - Id: "stale-claimant", - Name: "stale-claimant", - GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", - GPUAssignedAt: &assignedAt, - }})) - - claimed, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "other-instance", "/sys/bus/pci/devices/0000:82:00.4") - require.NoError(t, err) - assert.False(t, claimed) -} - -func TestVGPUAssignmentClaimedByLiveInstanceGracesRecentDeadPIDClaim(t *testing.T) { - m := &manager{paths: paths.New(t.TempDir())} - claimantID := "claimant-dead-pid" - require.NoError(t, m.ensureDirectories(claimantID)) - deadPID := 1<<22 - 1 - require.False(t, ProcessExists(deadPID)) - assignedAt := time.Now().UTC() - require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ - Id: claimantID, - HypervisorProcessIdentity: HypervisorProcessIdentity{HypervisorPID: &deadPID}, - GPUFramework: devices.VGPUFrameworkVendorVFIO, - GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", - GPUAssignedAt: &assignedAt, - }})) - - _, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "requester", "/sys/bus/pci/devices/0000:82:00.4") - require.Error(t, err) - - stale := assignedAt.Add(-2 * VGPUAssignmentStartupGracePeriod) - meta, err := m.loadMetadata(claimantID) - require.NoError(t, err) - meta.GPUAssignedAt = &stale - require.NoError(t, m.saveMetadata(meta)) - - claimed, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "requester", "/sys/bus/pci/devices/0000:82:00.4") - require.NoError(t, err) - assert.False(t, claimed) -} - -func TestVGPUAssignmentClaimedByLiveInstanceIgnoresDeadClaim(t *testing.T) { - t.Parallel() - - m := &manager{paths: paths.New(t.TempDir())} - require.NoError(t, m.ensureDirectories("dead-claimant")) + const devicePath = "/sys/bus/pci/devices/0000:82:00.4" deadPID := 1 << 30 - require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ - Id: "dead-claimant", - Name: "dead-claimant", - GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", - HypervisorProcessIdentity: HypervisorProcessIdentity{HypervisorPID: &deadPID}, - }})) - - claimed, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "other-instance", "/sys/bus/pci/devices/0000:82:00.4") - require.NoError(t, err) - assert.False(t, claimed, "a claim whose hypervisor is gone must not block the release") + require.False(t, ProcessExists(deadPID)) + recent := time.Now().UTC() + stale := recent.Add(-VGPUAssignmentStartupGracePeriod - time.Minute) + tests := []struct { + name string + assignedAt *time.Time + pid *int + wantErr string + }{ + {name: "recent without PID", assignedAt: &recent, wantErr: "no persisted hypervisor PID"}, + {name: "stale without PID", assignedAt: &stale}, + {name: "recent dead PID", assignedAt: &recent, pid: &deadPID, wantErr: "recorded hypervisor is not running"}, + {name: "legacy dead PID", pid: &deadPID}, + } + for i, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + m := &manager{paths: paths.New(t.TempDir())} + id := fmt.Sprintf("claimant-%d", i) + require.NoError(t, m.ensureDirectories(id)) + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: id, + GPUFramework: devices.VGPUFrameworkVendorVFIO, + GPUDevicePath: devicePath, + GPUAssignedAt: tt.assignedAt, + HypervisorProcessIdentity: HypervisorProcessIdentity{HypervisorPID: tt.pid}, + }})) + + claimed, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "requester", devicePath) + if tt.wantErr != "" { + require.ErrorContains(t, err, tt.wantErr) + return + } + require.NoError(t, err) + assert.False(t, claimed) + }) + } } func TestReleaseStoredVGPUSkipsClaimScanForMdev(t *testing.T) { From 33f58e10ee89ee69c67ff9b699b34efc2e2127bb Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Wed, 26 Aug 2026 17:40:31 +0000 Subject: [PATCH 71/76] Add counters for vGPU cleanup failure paths A wedged or leaked VF presents as reduced GPU capacity while /resources still reports full capacity, and reconcile, stale-release, and retention failures were visible only as log lines. Count failed reconcile stages, failed stale releases, and retained assignments (by operation and whether the retention record persisted) so sustained failure can alert. --- lib/instances/metrics.go | 81 +++++++++++++++++++++++++++++++++ lib/instances/metrics_test.go | 45 ++++++++++++++++++ lib/instances/start.go | 3 ++ lib/instances/vgpu_reconcile.go | 3 ++ lib/instances/vgpu_retention.go | 3 ++ 5 files changed, 135 insertions(+) diff --git a/lib/instances/metrics.go b/lib/instances/metrics.go index 1ada5ac1e..a380ff09f 100644 --- a/lib/instances/metrics.go +++ b/lib/instances/metrics.go @@ -73,6 +73,20 @@ type lifecycleEventDropReason string const lifecycleEventDropReasonBufferFull lifecycleEventDropReason = "buffer_full" +type vgpuReconcileStage string + +const ( + vgpuReconcileStageListInstances vgpuReconcileStage = "list_instances" + vgpuReconcileStageReconcileDevices vgpuReconcileStage = "reconcile_devices" +) + +type vgpuRetentionOperation string + +const ( + vgpuRetentionOperationCreate vgpuRetentionOperation = "create" + vgpuRetentionOperationStart vgpuRetentionOperation = "start" +) + // Metrics holds the metrics instruments for instance operations. type Metrics struct { createDuration metric.Float64Histogram @@ -94,6 +108,9 @@ type Metrics struct { lifecycleEventsDroppedTotal metric.Int64Counter forkMemFileShareFallbacksTotal metric.Int64Counter ttlReaperDeletionsTotal metric.Int64Counter + vgpuReconcileFailuresTotal metric.Int64Counter + vgpuStaleReleaseFailuresTotal metric.Int64Counter + vgpuAssignmentsRetainedTotal metric.Int64Counter tracer trace.Tracer } @@ -270,6 +287,30 @@ func newInstanceMetrics(meter metric.Meter, tracer trace.Tracer, m *manager) (*M return nil, err } + vgpuReconcileFailuresTotal, err := meter.Int64Counter( + "hypeman_instances_vgpu_reconcile_failures_total", + metric.WithDescription("Total number of vGPU reconcile pass stages that failed, leaving stale assignments or device leftovers allocated while /resources still advertises the capacity"), + ) + if err != nil { + return nil, err + } + + vgpuStaleReleaseFailuresTotal, err := meter.Int64Counter( + "hypeman_instances_vgpu_stale_release_failures_total", + metric.WithDescription("Total number of stale vGPU assignment releases that failed, keeping the VF allocated until a later reconcile pass succeeds"), + ) + if err != nil { + return nil, err + } + + vgpuAssignmentsRetainedTotal, err := meter.Int64Counter( + "hypeman_instances_vgpu_assignments_retained_total", + metric.WithDescription("Total number of failed rollbacks that left a vGPU assignment behind, by whether the retention record the periodic reconcile needs was persisted"), + ) + if err != nil { + return nil, err + } + // Register observable gauge for instance counts by state instancesTotal, err := meter.Int64ObservableGauge( "hypeman_instances_total", @@ -464,6 +505,9 @@ func newInstanceMetrics(meter metric.Meter, tracer trace.Tracer, m *manager) (*M lifecycleEventsDroppedTotal: lifecycleEventsDroppedTotal, forkMemFileShareFallbacksTotal: forkMemFileShareFallbacksTotal, ttlReaperDeletionsTotal: ttlReaperDeletionsTotal, + vgpuReconcileFailuresTotal: vgpuReconcileFailuresTotal, + vgpuStaleReleaseFailuresTotal: vgpuStaleReleaseFailuresTotal, + vgpuAssignmentsRetainedTotal: vgpuAssignmentsRetainedTotal, tracer: tracer, }, nil } @@ -563,6 +607,43 @@ func (m *manager) recordTimeToRunning(ctx context.Context, stored *StoredMetadat m.metrics.timeToRunning.Record(ctx, duration, metric.WithAttributes(attrs...)) } +// recordVGPUReconcileFailure records a vGPU reconcile stage failing: stale +// assignments or device leftovers stay allocated (capacity silently reduced +// while /resources reports full) until a later pass succeeds, so sustained +// failure must be alertable beyond a log line. +func (m *manager) recordVGPUReconcileFailure(ctx context.Context, stage vgpuReconcileStage) { + if m.metrics == nil { + return + } + m.metrics.vgpuReconcileFailuresTotal.Add(ctx, 1, metric.WithAttributes( + attribute.String("stage", string(stage)), + )) +} + +// recordVGPUStaleReleaseFailure records a stale vGPU release failing: the VF +// stays allocated while /resources still advertises it, so a wedged release +// that fails every pass must be alertable beyond a log line. +func (m *manager) recordVGPUStaleReleaseFailure(ctx context.Context) { + if m.metrics == nil { + return + } + m.metrics.vgpuStaleReleaseFailuresTotal.Add(ctx, 1) +} + +// recordVGPURetainedAssignment records a failed rollback leaving a vGPU +// assignment behind. An unpersisted retention record has no metadata claim, +// so the device is only recovered by the periodic reconcile sweep; either way +// the VF is unavailable while /resources still advertises it. +func (m *manager) recordVGPURetainedAssignment(ctx context.Context, operation vgpuRetentionOperation, persisted bool) { + if m.metrics == nil { + return + } + m.metrics.vgpuAssignmentsRetainedTotal.Add(ctx, 1, metric.WithAttributes( + attribute.String("operation", string(operation)), + attribute.String("persisted", strconv.FormatBool(persisted)), + )) +} + // recordStateTransition records a state transition with hypervisor label. func (m *manager) recordStateTransition(ctx context.Context, fromState, toState string, hvType hypervisor.Type) { if m.metrics == nil { diff --git a/lib/instances/metrics_test.go b/lib/instances/metrics_test.go index 811571e36..525002cbb 100644 --- a/lib/instances/metrics_test.go +++ b/lib/instances/metrics_test.go @@ -2,6 +2,7 @@ package instances import ( "context" + "errors" "os" "path/filepath" "testing" @@ -556,6 +557,50 @@ func TestEnsureSnapshotMemoryReadySkipsPendingCompressionWithoutPreemptionMetric assert.False(t, metricExists(rm, "hypeman_snapshot_compression_preemptions_total"), "pending-delay cancellation should not record a preemption") } +func TestVGPUReconcileFailureMetric_RecordStages(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("root bypasses directory permissions") + } + + reader := otelmetric.NewManualReader() + provider := otelmetric.NewMeterProvider(otelmetric.WithReader(reader)) + + m := &manager{ + paths: paths.New(t.TempDir()), + reconcileVGPUDevices: func(context.Context, map[string]struct{}, bool) error { + return errors.New("sweep failed") + }, + } + metrics, err := newInstanceMetrics(provider.Meter("test"), nil, m) + require.NoError(t, err) + m.metrics = metrics + + const id = "unreadable" + require.NoError(t, m.ensureDirectories(id)) + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{Id: id}})) + instanceDir := filepath.Dir(m.paths.InstanceMetadata(id)) + require.NoError(t, os.Chmod(instanceDir, 0o000)) + t.Cleanup(func() { _ = os.Chmod(instanceDir, 0o755) }) + + m.ReconcileVGPUs(t.Context()) + + var rm metricdata.ResourceMetrics + require.NoError(t, reader.Collect(t.Context(), &rm)) + + failuresMetric := findMetric(t, rm, "hypeman_instances_vgpu_reconcile_failures_total") + failures, ok := failuresMetric.Data.(metricdata.Sum[int64]) + require.True(t, ok) + require.Len(t, failures.DataPoints, 2) + for _, point := range failures.DataPoints { + switch metricLabel(t, point.Attributes, "stage") { + case "list_instances", "reconcile_devices": + assert.Equal(t, int64(1), point.Value) + default: + t.Fatalf("unexpected reconcile failure stage datapoint: %s", metricLabel(t, point.Attributes, "stage")) + } + } +} + func assertMetricNames(t *testing.T, rm metricdata.ResourceMetrics, expected []string) { t.Helper() diff --git a/lib/instances/start.go b/lib/instances/start.go index 19bb64697..ea8c2a6f4 100644 --- a/lib/instances/start.go +++ b/lib/instances/start.go @@ -183,9 +183,11 @@ func (m *manager) startInstance( setStoredVGPUDevice(&retentionMeta.StoredMetadata, pendingDevice, assignedAt) wrapped := fmt.Errorf("create vGPU for profile %s: %w", stored.GPUProfile, err) if saveErr := m.saveMetadata(&retentionMeta); saveErr != nil { + m.recordVGPURetainedAssignment(ctx, vgpuRetentionOperationStart, false) log.ErrorContext(ctx, "failed to retain vGPU assignment after create rollback failure", "instance_id", id, "error", saveErr) return nil, &VGPUCleanupPendingError{InstanceID: id, Err: fmt.Errorf("%w; retain assignment: %v", wrapped, saveErr)} } + m.recordVGPURetainedAssignment(ctx, vgpuRetentionOperationStart, true) return nil, &VGPUCleanupPendingError{InstanceID: id, Retained: true, Err: wrapped} } return nil, fmt.Errorf("create vGPU for profile %s: %w", stored.GPUProfile, err) @@ -198,6 +200,7 @@ func (m *manager) startInstance( retained, persisted := m.cleanupStartVGPU(ctx, id, device, assignedAt, rollbackMeta) if retained { retention.markRetained(persisted) + m.recordVGPURetainedAssignment(ctx, vgpuRetentionOperationStart, persisted) } }) if err := m.saveMetadata(meta); err != nil { diff --git a/lib/instances/vgpu_reconcile.go b/lib/instances/vgpu_reconcile.go index 5cfb43d4c..9ca8cef72 100644 --- a/lib/instances/vgpu_reconcile.go +++ b/lib/instances/vgpu_reconcile.go @@ -56,6 +56,7 @@ func (m *manager) ReconcileVGPUs(ctx context.Context) { protected, err := m.reconcileVGPUAssignments(ctx) sweepVendorVFIO := err == nil if err != nil { + m.recordVGPUReconcileFailure(ctx, vgpuReconcileStageListInstances) log.ErrorContext(ctx, "failed to list instances for vGPU reconcile protection; skipping vendor VFIO sweep until the next pass, mdev reconcile still runs", "error", err) protected = make(map[string]struct{}) } @@ -64,6 +65,7 @@ func (m *manager) ReconcileVGPUs(ctx context.Context) { reconcileDevices = devices.ReconcileVGPUs } if err := reconcileDevices(ctx, protected, sweepVendorVFIO); err != nil { + m.recordVGPUReconcileFailure(ctx, vgpuReconcileStageReconcileDevices) log.WarnContext(ctx, "failed to reconcile vGPU devices", "error", err) } } @@ -125,6 +127,7 @@ func (m *manager) releaseStaleVGPUAssignment(ctx context.Context, id string) { return } if err := m.releaseStoredVGPU(ctx, stored); err != nil { + m.recordVGPUStaleReleaseFailure(ctx) log.WarnContext(ctx, "failed to release stale vGPU assignment; retrying on the next reconcile pass", "instance_id", id, "device_path", path, "error", err) return } diff --git a/lib/instances/vgpu_retention.go b/lib/instances/vgpu_retention.go index e630484ae..691ae5eaa 100644 --- a/lib/instances/vgpu_retention.go +++ b/lib/instances/vgpu_retention.go @@ -61,6 +61,9 @@ func (m *manager) persistVGPURetention(ctx context.Context, retention *vgpuReten id := retention.instanceID retainedVGPU := retention.stub log := logger.FromContext(ctx) + defer func() { + m.recordVGPURetainedAssignment(ctx, vgpuRetentionOperationCreate, retention.persisted) + }() // When the retention record is lost, the assignment has no metadata claim // left; the periodic vGPU reconcile sweeps the device once it is free. retentionSurvives := func() bool { From ea55f132937e78518c2085a75c848f70dbc80c65 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Wed, 26 Aug 2026 18:28:01 +0000 Subject: [PATCH 72/76] Simplify vGPU lifecycle cleanup --- lib/devices/vgpu_linux.go | 28 +-- lib/devices/vgpu_linux_test.go | 16 -- lib/instances/lifecycle_noop_test.go | 36 --- lib/instances/manager.go | 10 +- lib/instances/metrics.go | 11 - lib/instances/process_identity.go | 8 +- lib/instances/process_identity_linux_test.go | 4 +- lib/instances/query_test.go | 24 +- lib/instances/vgpu.go | 96 ++------ lib/instances/vgpu_reconcile.go | 18 +- lib/instances/vgpu_reconcile_linux_test.go | 6 +- lib/instances/vgpu_reconcile_test.go | 2 +- lib/instances/vgpu_retention.go | 42 +--- lib/instances/vgpu_test.go | 233 +++++-------------- 14 files changed, 113 insertions(+), 421 deletions(-) diff --git a/lib/devices/vgpu_linux.go b/lib/devices/vgpu_linux.go index 72827f7b2..36bf86c9e 100644 --- a/lib/devices/vgpu_linux.go +++ b/lib/devices/vgpu_linux.go @@ -111,33 +111,13 @@ func ReconcileVGPUs(ctx context.Context, protectedDevicePaths map[string]struct{ return err } - return reconcileDiscoveredVGPUs( - ctx, - framework, - protectedDevicePaths, - sweepVendorVFIO, - func(ctx context.Context) error { return ReconcileMdevs(ctx, nil) }, - hostVendorVFIO.reconcile, - ) -} - -func reconcileDiscoveredVGPUs( - ctx context.Context, - framework VGPUFramework, - protectedDevicePaths map[string]struct{}, - sweepVendorVFIO bool, - reconcileMdev func(context.Context) error, - reconcileVendorVFIO func(context.Context, map[string]struct{}) error, -) error { switch framework { case VGPUFrameworkMdev: - return reconcileMdev(ctx) + return ReconcileMdevs(ctx, nil) case VGPUFrameworkVendorVFIO: - if !sweepVendorVFIO { - return nil + if sweepVendorVFIO { + return hostVendorVFIO.reconcile(ctx, protectedDevicePaths) } - return reconcileVendorVFIO(ctx, protectedDevicePaths) - default: - return nil } + return nil } diff --git a/lib/devices/vgpu_linux_test.go b/lib/devices/vgpu_linux_test.go index 32f9a30ea..7b03d9c26 100644 --- a/lib/devices/vgpu_linux_test.go +++ b/lib/devices/vgpu_linux_test.go @@ -3,7 +3,6 @@ package devices import ( - "context" "errors" "os" "path/filepath" @@ -13,21 +12,6 @@ import ( "github.com/stretchr/testify/require" ) -func TestReconcileDiscoveredVGPUsControlsVendorSweep(t *testing.T) { - protected := make(map[string]struct{}) - vendorCalls := 0 - reconcileVendor := func(context.Context, map[string]struct{}) error { - vendorCalls++ - return nil - } - - require.NoError(t, reconcileDiscoveredVGPUs(context.Background(), VGPUFrameworkVendorVFIO, protected, false, nil, reconcileVendor)) - assert.Zero(t, vendorCalls) - - require.NoError(t, reconcileDiscoveredVGPUs(context.Background(), VGPUFrameworkVendorVFIO, protected, true, nil, reconcileVendor)) - assert.Equal(t, 1, vendorCalls) -} - func TestDiscoverVGPUWithPropagatesMdevError(t *testing.T) { t.Parallel() diff --git a/lib/instances/lifecycle_noop_test.go b/lib/instances/lifecycle_noop_test.go index 85083766b..77d15d733 100644 --- a/lib/instances/lifecycle_noop_test.go +++ b/lib/instances/lifecycle_noop_test.go @@ -150,24 +150,6 @@ func TestLifecycleNoopStandbyWithOptionsStillRejectsStandbyInstance(t *testing.T assertNoLifecycleEvent(t, events) } -func TestDeleteContinuesWhenVGPUReleaseFails(t *testing.T) { - m, id := newLifecycleNoopManagerWithInstance(t, StateStopped, time.Now().UTC()) - meta, err := m.loadMetadata(id) - require.NoError(t, err) - meta.GPUProfile = "NVIDIA L40S-2Q" - meta.GPUFramework = devices.VGPUFramework("future-framework") - meta.GPUDevicePath = "/sys/bus/pci/devices/0000:82:00.4" - require.NoError(t, m.saveMetadata(meta)) - - // A failed release is logged and the delete continues, matching the - // pre-refactor contract; the leaked assignment is recovered by the - // periodic vGPU reconcile. - require.NoError(t, m.DeleteInstance(context.Background(), id)) - - _, err = m.loadMetadata(id) - require.Error(t, err, "instance data must be deleted despite the failed release") -} - func TestDeletePersistsVGPUReleaseBeforeTeardown(t *testing.T) { m, id := newLifecycleNoopManagerWithInstance(t, StateStopped, time.Now().UTC()) var persisted *metadata @@ -360,24 +342,6 @@ func TestStopStoppedInstanceLeavesVGPUForReconcile(t *testing.T) { assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", stored.GPUDevicePath) } -func TestReconcileVGPUReleaseFailureKeepsStoppedInstanceUsable(t *testing.T) { - m, id := newLifecycleNoopManagerWithInstance(t, StateStopped, time.Now().UTC()) - m.reconcileVGPUDevices = func(context.Context, map[string]struct{}, bool) error { return nil } - meta, err := m.loadMetadata(id) - require.NoError(t, err) - meta.GPUProfile = "NVIDIA L40S-2Q" - meta.GPUFramework = devices.VGPUFramework("future-framework") - meta.GPUDevicePath = "/sys/bus/pci/devices/0000:82:00.4" - require.NoError(t, m.saveMetadata(meta)) - - m.ReconcileVGPUs(context.Background()) - - stored, err := m.loadMetadata(id) - require.NoError(t, err) - assert.Equal(t, devices.VGPUFramework("future-framework"), stored.GPUFramework) - assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", stored.GPUDevicePath) -} - // recordingDeviceManager is a devices.Manager stub that records passthrough // teardown calls. Only the methods delete exercises are implemented. type recordingDeviceManager struct { diff --git a/lib/instances/manager.go b/lib/instances/manager.go index 8c298ed24..d44ef3014 100644 --- a/lib/instances/manager.go +++ b/lib/instances/manager.go @@ -219,7 +219,7 @@ type manager struct { vgpuReconcileInterval time.Duration discoverVGPU func() (devices.VGPUFramework, []devices.VirtualFunction, error) - vgpuInitTermGrace time.Duration + vfioTermGrace time.Duration // Hypervisor support vmStarters map[hypervisor.Type]hypervisor.VMStarter @@ -293,8 +293,6 @@ func NewManagerWithConfigE(p *paths.Paths, imageManager images.Manager, systemMa defaultHypervisor: defaultHypervisor, now: time.Now, writeFile: os.WriteFile, - createVGPU: devices.CreateVGPU, - destroyVGPU: devices.DestroyVGPU, meter: meter, tracer: tracer, guestMemoryPolicy: policy, @@ -740,12 +738,12 @@ func (m *manager) DefaultHypervisor() hypervisor.Type { return m.defaultHypervisor } -func (m *manager) listInstancesForReconcile(ctx context.Context) ([]Instance, error) { +func (m *manager) listMetadataForReconcile() ([]StoredMetadata, error) { files, err := m.listMetadataFilesStrict() if err != nil { return nil, err } - result := make([]Instance, 0, len(files)) + result := make([]StoredMetadata, 0, len(files)) for _, file := range files { id := filepath.Base(filepath.Dir(file)) meta, err := m.loadMetadata(id) @@ -755,7 +753,7 @@ func (m *manager) listInstancesForReconcile(ctx context.Context) ([]Instance, er } return nil, fmt.Errorf("load metadata for instance %s: %w", id, err) } - result = append(result, Instance{StoredMetadata: meta.StoredMetadata}) + result = append(result, meta.StoredMetadata) } return result, nil } diff --git a/lib/instances/metrics.go b/lib/instances/metrics.go index a380ff09f..e14deb49b 100644 --- a/lib/instances/metrics.go +++ b/lib/instances/metrics.go @@ -607,10 +607,6 @@ func (m *manager) recordTimeToRunning(ctx context.Context, stored *StoredMetadat m.metrics.timeToRunning.Record(ctx, duration, metric.WithAttributes(attrs...)) } -// recordVGPUReconcileFailure records a vGPU reconcile stage failing: stale -// assignments or device leftovers stay allocated (capacity silently reduced -// while /resources reports full) until a later pass succeeds, so sustained -// failure must be alertable beyond a log line. func (m *manager) recordVGPUReconcileFailure(ctx context.Context, stage vgpuReconcileStage) { if m.metrics == nil { return @@ -620,9 +616,6 @@ func (m *manager) recordVGPUReconcileFailure(ctx context.Context, stage vgpuReco )) } -// recordVGPUStaleReleaseFailure records a stale vGPU release failing: the VF -// stays allocated while /resources still advertises it, so a wedged release -// that fails every pass must be alertable beyond a log line. func (m *manager) recordVGPUStaleReleaseFailure(ctx context.Context) { if m.metrics == nil { return @@ -630,10 +623,6 @@ func (m *manager) recordVGPUStaleReleaseFailure(ctx context.Context) { m.metrics.vgpuStaleReleaseFailuresTotal.Add(ctx, 1) } -// recordVGPURetainedAssignment records a failed rollback leaving a vGPU -// assignment behind. An unpersisted retention record has no metadata claim, -// so the device is only recovered by the periodic reconcile sweep; either way -// the VF is unavailable while /resources still advertises it. func (m *manager) recordVGPURetainedAssignment(ctx context.Context, operation vgpuRetentionOperation, persisted bool) { if m.metrics == nil { return diff --git a/lib/instances/process_identity.go b/lib/instances/process_identity.go index 02f43fcda..7e8500402 100644 --- a/lib/instances/process_identity.go +++ b/lib/instances/process_identity.go @@ -27,9 +27,9 @@ const linuxBootIDPath = "/proc/sys/kernel/random/boot_id" // does not unstick it, so the wait is short to keep stop and delete fast. const hypervisorSIGKILLWaitTimeout = 2 * time.Second -func (m *manager) vgpuTermGrace() time.Duration { - if m.vgpuInitTermGrace > 0 { - return m.vgpuInitTermGrace +func (m *manager) vfioTerminationGrace() time.Duration { + if m.vfioTermGrace > 0 { + return m.vfioTermGrace } return hypervisor.VFIOTermGrace } @@ -37,7 +37,7 @@ func (m *manager) vgpuTermGrace() time.Duration { // SIGKILL during guest driver init can wedge a VF until the parent GPU is reset. func (m *manager) terminateThenKill(ctx context.Context, inst *Instance, pid int) error { if inst.GPUProfile != "" || len(inst.Devices) > 0 { - if syscall.Kill(pid, syscall.SIGTERM) == nil && WaitForProcessExit(pid, m.vgpuTermGrace()) { + if syscall.Kill(pid, syscall.SIGTERM) == nil && WaitForProcessExit(pid, m.vfioTerminationGrace()) { return nil } logger.FromContext(ctx).WarnContext(ctx, "hypervisor with VFIO devices did not exit on SIGTERM; hard-killing, device may wedge if the guest driver was initializing", diff --git a/lib/instances/process_identity_linux_test.go b/lib/instances/process_identity_linux_test.go index b208a6ef3..2ec201450 100644 --- a/lib/instances/process_identity_linux_test.go +++ b/lib/instances/process_identity_linux_test.go @@ -546,7 +546,7 @@ func TestVGPUAssignmentClaimedByLiveInstanceProtectsReusedPIDClaim(t *testing.T) SocketPath: socketPath, }})) - claimed, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "other-instance", devicePath) + claimed, err := m.vgpuAssignmentClaimedByLiveInstance("other-instance", devicePath) require.NoError(t, err) assert.True(t, claimed) } @@ -740,7 +740,7 @@ func TestKillHypervisorEscalatesToSIGKILLWhenSIGTERMIgnored(t *testing.T) { pid, identity := startTrapProcess(t, "") socketPath := filepath.Join(t.TempDir(), "missing.sock") - m := &manager{vgpuInitTermGrace: 50 * time.Millisecond} + m := &manager{vfioTermGrace: 50 * time.Millisecond} require.NoError(t, m.killHypervisor(context.Background(), &Instance{ State: StateInitializing, StoredMetadata: StoredMetadata{ diff --git a/lib/instances/query_test.go b/lib/instances/query_test.go index 795b00baf..222e85a30 100644 --- a/lib/instances/query_test.go +++ b/lib/instances/query_test.go @@ -14,7 +14,7 @@ import ( "github.com/stretchr/testify/require" ) -func TestListInstancesForReconcileFailsOnInvalidMetadata(t *testing.T) { +func TestListMetadataForReconcileFailsOnInvalidMetadata(t *testing.T) { m := &manager{paths: paths.New(t.TempDir())} require.NoError(t, m.ensureDirectories("valid")) @@ -31,18 +31,18 @@ func TestListInstancesForReconcileFailsOnInvalidMetadata(t *testing.T) { require.NoError(t, err) require.Len(t, listed, 1) - _, err = m.listInstancesForReconcile(context.Background()) + _, err = m.listMetadataForReconcile() require.Error(t, err) assert.ErrorContains(t, err, "load metadata for instance invalid") require.NoError(t, os.Remove(m.paths.InstanceMetadata("invalid"))) - listed, err = m.listInstancesForReconcile(context.Background()) + metadata, err := m.listMetadataForReconcile() require.NoError(t, err) - require.Len(t, listed, 1) - assert.Equal(t, "valid", listed[0].Id) + require.Len(t, metadata, 1) + assert.Equal(t, "valid", metadata[0].Id) } -func TestListInstancesForReconcileSkipsInstanceDeletedDuringListing(t *testing.T) { +func TestListMetadataForReconcileSkipsInstanceDeletedDuringListing(t *testing.T) { m := &manager{paths: paths.New(t.TempDir())} for _, id := range []string{"aaa-ghost", "zzz-live"} { @@ -57,13 +57,13 @@ func TestListInstancesForReconcileSkipsInstanceDeletedDuringListing(t *testing.T unlock := hypervisor.LockSnapshotSourceAliasMutation() type result struct { - listed []Instance - err error + metadata []StoredMetadata + err error } done := make(chan result, 1) go func() { - listed, err := m.listInstancesForReconcile(context.Background()) - done <- result{listed, err} + metadata, err := m.listMetadataForReconcile() + done <- result{metadata, err} }() time.Sleep(100 * time.Millisecond) require.NoError(t, os.Remove(m.paths.InstanceMetadata("aaa-ghost"))) @@ -71,8 +71,8 @@ func TestListInstancesForReconcileSkipsInstanceDeletedDuringListing(t *testing.T res := <-done require.NoError(t, res.err) - require.Len(t, res.listed, 1) - assert.Equal(t, "zzz-live", res.listed[0].Id) + require.Len(t, res.metadata, 1) + assert.Equal(t, "zzz-live", res.metadata[0].Id) } func TestParseExitSentinelLine(t *testing.T) { diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index 37127e921..7b70e807c 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -11,10 +11,6 @@ import ( "github.com/kernel/hypeman/lib/logger" ) -// VGPUAssignmentStartupGracePeriod bounds how long an assignment without a -// persisted hypervisor PID is treated as potentially live. -const VGPUAssignmentStartupGracePeriod = devices.VGPUAssignmentGracePeriod - // VGPUCleanupPendingError reports a failed rollback that left a vGPU assigned. type VGPUCleanupPendingError struct { InstanceID string @@ -49,18 +45,9 @@ func vgpuDevicePendingCleanup(err error) (*devices.VGPUDevice, bool) { return &pending.Device, true } -func vgpuAssignmentLiveness(stored *StoredMetadata, now time.Time, livePID bool) (live bool, graceRemaining time.Duration) { - if livePID { - return true, 0 - } - if stored.GPUAssignedAt == nil { - return false, 0 - } - remaining := VGPUAssignmentStartupGracePeriod - now.Sub(*stored.GPUAssignedAt) - if remaining <= 0 { - return false, 0 - } - return true, remaining +func vgpuAssignmentMayBeLive(stored *StoredMetadata, now time.Time, hypervisorLive bool) bool { + return hypervisorLive || + stored.GPUAssignedAt != nil && now.Sub(*stored.GPUAssignedAt) < devices.VGPUAssignmentGracePeriod } func (m *manager) destroyVGPUAssignment(ctx context.Context, assignment devices.VGPUAssignment) error { @@ -87,61 +74,32 @@ func clearStoredVGPUDevice(stored *StoredMetadata) { func (m *manager) cleanupStartVGPU(ctx context.Context, instanceID string, device *devices.VGPUDevice, assignedAt time.Time, rollbackMeta metadata) (retained, persisted bool) { logger.FromContext(ctx).DebugContext(ctx, "destroying vGPU on cleanup", "instance_id", instanceID, "uuid", device.MdevUUID) - assignment := devices.VGPUAssignment{ + releaseErr := m.destroyVGPUAssignment(ctx, devices.VGPUAssignment{ Framework: device.Framework, DevicePath: device.SysfsPath, MdevUUID: device.MdevUUID, InstanceID: instanceID, - } - cleanupMeta, err := m.loadMetadata(instanceID) - if err != nil { - logger.FromContext(ctx).WarnContext(ctx, "failed to load current metadata for vGPU cleanup; restoring rollback snapshot", "instance_id", instanceID, "error", err) - cleanupMeta = &rollbackMeta - } else { - restoreStartMutatedFields(&cleanupMeta.StoredMetadata, &rollbackMeta.StoredMetadata) - } - releaseErr := m.destroyVGPUAssignment(ctx, assignment) + }) if releaseErr != nil { logger.FromContext(ctx).WarnContext(ctx, "failed to destroy vGPU on cleanup", "instance_id", instanceID, "uuid", device.MdevUUID, "error", releaseErr) - setStoredVGPUDevice(&cleanupMeta.StoredMetadata, device, assignedAt) + setStoredVGPUDevice(&rollbackMeta.StoredMetadata, device, assignedAt) retained = true } - if err := m.saveMetadata(cleanupMeta); err != nil { + if err := m.saveMetadata(&rollbackMeta); err != nil { message := "failed to save metadata after vGPU cleanup" - if releaseErr != nil { + if retained { message = "failed to retain vGPU assignment metadata after cleanup failure" } logger.FromContext(ctx).ErrorContext(ctx, message, "instance_id", instanceID, "error", err) if !retained { return false, false } - if meta, loadErr := m.loadMetadata(instanceID); loadErr == nil && storedVGPUDevicePath(&meta.StoredMetadata) == device.SysfsPath { - return true, true - } - return true, false + meta, loadErr := m.loadMetadata(instanceID) + return true, loadErr == nil && storedVGPUDevicePath(&meta.StoredMetadata) == device.SysfsPath } return retained, retained } -func restoreStartMutatedFields(dst, src *StoredMetadata) { - dst.HypervisorPID = src.HypervisorPID - dst.HypervisorStartTime = src.HypervisorStartTime - dst.HypervisorBootID = src.HypervisorBootID - dst.ExitCode = src.ExitCode - dst.ExitMessage = src.ExitMessage - dst.ProgramStartedAt = src.ProgramStartedAt - dst.GuestAgentReadyAt = src.GuestAgentReadyAt - dst.Entrypoint = src.Entrypoint - dst.Cmd = src.Cmd - dst.IP = src.IP - dst.MAC = src.MAC - dst.GPUFramework = src.GPUFramework - dst.GPUDevicePath = src.GPUDevicePath - dst.GPUMdevUUID = src.GPUMdevUUID - dst.GPUAssignedAt = src.GPUAssignedAt - dst.StartedAt = src.StartedAt -} - func (m *manager) releaseStoredVGPU(ctx context.Context, stored *StoredMetadata) error { path := storedVGPUDevicePath(stored) if path != "" { @@ -149,7 +107,7 @@ func (m *manager) releaseStoredVGPU(ctx context.Context, stored *StoredMetadata) claimed := false if stored.GPUFramework == devices.VGPUFrameworkVendorVFIO { var err error - claimed, err = m.vgpuAssignmentClaimedByLiveInstance(ctx, stored.Id, path) + claimed, err = m.vgpuAssignmentClaimedByLiveInstance(stored.Id, path) if err != nil { return err } @@ -173,42 +131,28 @@ func (m *manager) releaseStoredVGPU(ctx context.Context, stored *StoredMetadata) return nil } -func (m *manager) vgpuAssignmentClaimedByLiveInstance(ctx context.Context, excludeID, devicePath string) (bool, error) { - files, err := m.listMetadataFilesStrict() +func (m *manager) vgpuAssignmentClaimedByLiveInstance(excludeID, devicePath string) (bool, error) { + allMetadata, err := m.listMetadataForReconcile() if err != nil { return false, fmt.Errorf("list instances for vGPU release check: %w", err) } - for _, file := range files { - id := filepath.Base(filepath.Dir(file)) - if id == excludeID { - continue - } - meta, err := m.loadMetadata(id) - if err != nil { - if errors.Is(err, ErrNotFound) { - continue - } - return false, fmt.Errorf("load metadata for vGPU release check: instance %s: %w", id, err) - } - stored := &meta.StoredMetadata - if storedVGPUDevicePath(stored) != devicePath { + for i := range allMetadata { + stored := &allMetadata[i] + if stored.Id == excludeID || storedVGPUDevicePath(stored) != devicePath { continue } - // Resolve even without a persisted PID: the socket-ownership scan can - // still prove a claimant whose post-boot metadata save failed is live. pid, err := resolveLiveHypervisorPID(stored.HypervisorProcessIdentity, stored.SocketPath) if err != nil { - return false, fmt.Errorf("cannot confirm liveness of vGPU claimant %s on %s: %w", id, devicePath, err) + return false, fmt.Errorf("cannot confirm liveness of vGPU claimant %s on %s: %w", stored.Id, devicePath, err) } - _, remaining := vgpuAssignmentLiveness(stored, m.nowUTC(), pid > 0) if pid > 0 { return true, nil } - if remaining > 0 { + if vgpuAssignmentMayBeLive(stored, m.nowUTC(), false) { if stored.HypervisorPID == nil { - return false, fmt.Errorf("cannot confirm liveness of recent vGPU claimant %s on %s: no persisted hypervisor PID", id, devicePath) + return false, fmt.Errorf("cannot confirm liveness of recent vGPU claimant %s on %s: no persisted hypervisor PID", stored.Id, devicePath) } - return false, fmt.Errorf("cannot confirm liveness of recent vGPU claimant %s on %s: recorded hypervisor is not running", id, devicePath) + return false, fmt.Errorf("cannot confirm liveness of recent vGPU claimant %s on %s: recorded hypervisor is not running", stored.Id, devicePath) } } return false, nil diff --git a/lib/instances/vgpu_reconcile.go b/lib/instances/vgpu_reconcile.go index 9ca8cef72..066c9bdee 100644 --- a/lib/instances/vgpu_reconcile.go +++ b/lib/instances/vgpu_reconcile.go @@ -75,22 +75,18 @@ func (m *manager) ReconcileVGPUs(ctx context.Context) { // Listing fails closed: any unreadable metadata aborts the pass so the vendor // VFIO sweep cannot clear a VF whose claim it could not read. func (m *manager) reconcileVGPUAssignments(ctx context.Context) (map[string]struct{}, error) { - allInstances, err := m.listInstancesForReconcile(ctx) + allMetadata, err := m.listMetadataForReconcile() if err != nil { return nil, err } protected := make(map[string]struct{}) - for i := range allInstances { - stored := &allInstances[i].StoredMetadata + for i := range allMetadata { + stored := &allMetadata[i] if storedVGPUDevicePath(stored) == "" { continue } - // The socket-ownership check runs even without a persisted PID: a VMM - // whose post-boot metadata save failed still holds its control-socket - // listener, and releasing its device would tear the vGPU out from - // under a live VM. - livePID := hypervisorMayBeAlive(stored.HypervisorProcessIdentity, stored.SocketPath) - if live, _ := vgpuAssignmentLiveness(stored, m.nowUTC(), livePID); live { + hypervisorLive := hypervisorMayBeAlive(stored.HypervisorProcessIdentity, stored.SocketPath) + if vgpuAssignmentMayBeLive(stored, m.nowUTC(), hypervisorLive) { if stored.GPUDevicePath != "" { protected[stored.GPUDevicePath] = struct{}{} } @@ -122,8 +118,8 @@ func (m *manager) releaseStaleVGPUAssignment(ctx context.Context, id string) { if path == "" { return } - livePID := hypervisorMayBeAlive(stored.HypervisorProcessIdentity, stored.SocketPath) - if live, _ := vgpuAssignmentLiveness(stored, m.nowUTC(), livePID); live { + hypervisorLive := hypervisorMayBeAlive(stored.HypervisorProcessIdentity, stored.SocketPath) + if vgpuAssignmentMayBeLive(stored, m.nowUTC(), hypervisorLive) { return } if err := m.releaseStoredVGPU(ctx, stored); err != nil { diff --git a/lib/instances/vgpu_reconcile_linux_test.go b/lib/instances/vgpu_reconcile_linux_test.go index 8708a1dd7..553c64c15 100644 --- a/lib/instances/vgpu_reconcile_linux_test.go +++ b/lib/instances/vgpu_reconcile_linux_test.go @@ -15,10 +15,6 @@ import ( "github.com/stretchr/testify/require" ) -// A VMM whose post-boot metadata save failed has no persisted PID, but it -// still holds its control-socket listener. The reconciler must protect its -// assignment past the startup grace period instead of releasing the device -// out from under the live VM. func TestReconcileVGPUsProtectsSocketOwnerWithoutPersistedPID(t *testing.T) { t.Parallel() @@ -41,7 +37,7 @@ func TestReconcileVGPUsProtectsSocketOwnerWithoutPersistedPID(t *testing.T) { }, } const id = "pid-save-failed" - stale := time.Now().UTC().Add(-VGPUAssignmentStartupGracePeriod - time.Minute) + stale := time.Now().UTC().Add(-devices.VGPUAssignmentGracePeriod - time.Minute) require.NoError(t, m.ensureDirectories(id)) require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ Id: id, diff --git a/lib/instances/vgpu_reconcile_test.go b/lib/instances/vgpu_reconcile_test.go index 395010532..7a9b66731 100644 --- a/lib/instances/vgpu_reconcile_test.go +++ b/lib/instances/vgpu_reconcile_test.go @@ -22,7 +22,7 @@ func TestReconcileVGPUsBoundsStartupProtection(t *testing.T) { deadPID := dead.Process.Pid now := time.Now().UTC() recent := now.Add(-time.Minute) - stale := now.Add(-VGPUAssignmentStartupGracePeriod - time.Minute) + stale := now.Add(-devices.VGPUAssignmentGracePeriod - time.Minute) var protected map[string]struct{} var destroyed []devices.VGPUAssignment diff --git a/lib/instances/vgpu_retention.go b/lib/instances/vgpu_retention.go index 691ae5eaa..2804e8abf 100644 --- a/lib/instances/vgpu_retention.go +++ b/lib/instances/vgpu_retention.go @@ -2,13 +2,9 @@ package instances import ( "context" - "encoding/json" - "fmt" - "os" "time" "github.com/kernel/hypeman/lib/devices" - "github.com/kernel/hypeman/lib/hypervisor" "github.com/kernel/hypeman/lib/logger" ) @@ -46,8 +42,7 @@ func (r *vgpuRetention) wrapPending(err error) error { return &VGPUCleanupPendingError{InstanceID: r.instanceID, Retained: r.persisted, Err: err} } -// deferWrapPending must be deferred before cleanup so it observes retention -// state recorded by rollback. +// Defer before cleanup so rollback records retention before this wraps the error. func (r *vgpuRetention) deferWrapPending(retErr *error) { *retErr = r.wrapPending(*retErr) } @@ -64,21 +59,13 @@ func (m *manager) persistVGPURetention(ctx context.Context, retention *vgpuReten defer func() { m.recordVGPURetainedAssignment(ctx, vgpuRetentionOperationCreate, retention.persisted) }() - // When the retention record is lost, the assignment has no metadata claim - // left; the periodic vGPU reconcile sweeps the device once it is free. retentionSurvives := func() bool { meta, err := m.loadMetadata(id) if err == nil && storedVGPUDevicePath(&meta.StoredMetadata) != "" { - saveErr := m.saveVGPURetentionStub(retainedVGPU) - if saveErr == nil { - return true + if err := m.saveVGPURetentionStub(retainedVGPU); err != nil { + log.ErrorContext(ctx, "failed to replace surviving instance metadata with vGPU retention stub; preserving existing assignment claim", "instance_id", id, "error", err) } - log.ErrorContext(ctx, "failed to replace surviving instance metadata with vGPU retention stub", "instance_id", id, "error", saveErr) - overwriteErr := m.overwriteVGPURetentionStub(retainedVGPU) - if overwriteErr == nil { - return true - } - log.ErrorContext(ctx, "failed to overwrite surviving instance metadata with vGPU retention stub", "instance_id", id, "error", overwriteErr) + return true } if err := m.deleteInstanceData(id); err != nil { log.ErrorContext(ctx, "failed to delete stale instance data after retention failure", "instance_id", id, "error", err) @@ -127,24 +114,3 @@ func vgpuRetentionMetadata(source *StoredMetadata) *metadata { func (m *manager) saveVGPURetentionStub(source *StoredMetadata) error { return m.saveMetadata(vgpuRetentionMetadata(source)) } - -// overwriteVGPURetentionStub handles a surviving metadata file when its -// directory cannot create the temporary file used by saveMetadata. -func (m *manager) overwriteVGPURetentionStub(source *StoredMetadata) error { - retained := vgpuRetentionMetadata(source) - data, err := json.MarshalIndent(retained, "", " ") - if err != nil { - return fmt.Errorf("marshal metadata: %w", err) - } - unlockAliasReaders := hypervisor.LockSnapshotSourceAliasReaders() - defer unlockAliasReaders() - writeFile := m.writeFile - if writeFile == nil { - writeFile = os.WriteFile - } - if err := writeFile(m.paths.InstanceMetadata(source.Id), data, 0644); err != nil { - return fmt.Errorf("write metadata: %w", err) - } - m.syncAdmissionAllocation(retained) - return nil -} diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index 55f88f4d9..b575c7546 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -24,53 +24,51 @@ func persistTestVGPURetention(m *manager, ctx context.Context, id string, stub * return retention.persisted } -func retainedVGPUFromCreateErrorForTest(stub StoredMetadata, assignedAt time.Time, err error) *StoredMetadata { - retention := vgpuRetention{} - retention.retainFromCreateError(stub, assignedAt, err) - return retention.stub -} - func TestCleanupFailedCreateRetainsVGPUAssignment(t *testing.T) { t.Parallel() m := &manager{paths: paths.New(t.TempDir())} - assignedAt := time.Now().UTC() - stored := &StoredMetadata{ - Id: "failed-create", - Name: "failed-create", - GPUProfile: "NVIDIA L40S-2Q", - GPUFramework: devices.VGPUFrameworkVendorVFIO, - GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", - GPUMdevUUID: "mdev-uuid", - GPUAssignedAt: &assignedAt, + const id = "failed-create" + stub := StoredMetadata{ + Id: id, + Name: id, NetworkEnabled: true, IP: "192.0.2.1", Volumes: []VolumeAttachment{{VolumeID: "volume"}}, HypervisorType: "qemu", - DataDir: m.paths.InstanceDir("failed-create"), + DataDir: m.paths.InstanceDir(id), } - require.NoError(t, m.ensureDirectories(stored.Id)) - require.NoError(t, os.WriteFile(m.paths.InstanceOverlay(stored.Id), []byte("overlay"), 0o644)) - require.NoError(t, os.WriteFile(m.paths.InstanceConfigDisk(stored.Id), []byte("config"), 0o644)) - require.NoError(t, os.MkdirAll(m.paths.InstanceVolumeOverlaysDir(stored.Id), 0o755)) - require.NoError(t, os.WriteFile(m.paths.InstanceVolumeOverlay(stored.Id, "volume"), []byte("volume overlay"), 0o644)) - - assert.True(t, persistTestVGPURetention(m, context.Background(), stored.Id, stored)) - assert.NoFileExists(t, m.paths.InstanceOverlay(stored.Id)) - assert.NoFileExists(t, m.paths.InstanceConfigDisk(stored.Id)) - assert.NoDirExists(t, m.paths.InstanceVolumeOverlaysDir(stored.Id)) - - retained, err := m.loadMetadata(stored.Id) + device := devices.VGPUDevice{ + ProfileName: "NVIDIA L40S-2Q", + Framework: devices.VGPUFrameworkVendorVFIO, + SysfsPath: "/sys/bus/pci/devices/0000:82:00.4", + } + assignedAt := time.Now().UTC() + retention := vgpuRetention{instanceID: id} + retention.retainFromCreateError(stub, assignedAt, &devices.VGPUCreateCleanupPendingError{Device: device, Err: errors.New("rollback failed")}) + + require.NoError(t, m.ensureDirectories(id)) + require.NoError(t, os.WriteFile(m.paths.InstanceOverlay(id), []byte("overlay"), 0o644)) + require.NoError(t, os.WriteFile(m.paths.InstanceConfigDisk(id), []byte("config"), 0o644)) + require.NoError(t, os.MkdirAll(m.paths.InstanceVolumeOverlaysDir(id), 0o755)) + require.NoError(t, os.WriteFile(m.paths.InstanceVolumeOverlay(id, "volume"), []byte("volume overlay"), 0o644)) + + m.persistVGPURetention(context.Background(), &retention) + assert.True(t, retention.persisted) + assert.NoFileExists(t, m.paths.InstanceOverlay(id)) + assert.NoFileExists(t, m.paths.InstanceConfigDisk(id)) + assert.NoDirExists(t, m.paths.InstanceVolumeOverlaysDir(id)) + + retained, err := m.loadMetadata(id) require.NoError(t, err) - assert.Equal(t, stored.Id, retained.Id) - assert.Equal(t, stored.GPUFramework, retained.GPUFramework) - assert.Equal(t, stored.GPUDevicePath, retained.GPUDevicePath) - assert.Equal(t, stored.GPUMdevUUID, retained.GPUMdevUUID) - assert.Equal(t, stored.GPUAssignedAt, retained.GPUAssignedAt) - assert.Equal(t, stored.Name, retained.Name) - assert.Equal(t, stored.GPUProfile, retained.GPUProfile) - assert.Equal(t, stored.HypervisorType, retained.HypervisorType) - assert.Equal(t, stored.DataDir, retained.DataDir) + assert.Equal(t, id, retained.Id) + assert.Equal(t, device.Framework, retained.GPUFramework) + assert.Equal(t, device.SysfsPath, retained.GPUDevicePath) + assert.Equal(t, assignedAt, *retained.GPUAssignedAt) + assert.Equal(t, stub.Name, retained.Name) + assert.Equal(t, device.ProfileName, retained.GPUProfile) + assert.Equal(t, stub.HypervisorType, retained.HypervisorType) + assert.Equal(t, stub.DataDir, retained.DataDir) assert.False(t, retained.NetworkEnabled) assert.Empty(t, retained.IP) assert.Empty(t, retained.Volumes) @@ -109,7 +107,7 @@ func TestCleanupFailedCreateReportsUnpersistedRetention(t *testing.T) { require.Error(t, err, "the lost retention leaves no metadata claim, so the periodic sweep releases the VF") } -func TestCleanupFailedCreateRewritesFullMetadataWhenDirectoryIsReadOnly(t *testing.T) { +func TestCleanupFailedCreatePreservesClaimWhenStubSaveFails(t *testing.T) { if os.Geteuid() == 0 { t.Skip("root bypasses directory permissions") } @@ -119,6 +117,7 @@ func TestCleanupFailedCreateRewritesFullMetadataWhenDirectoryIsReadOnly(t *testi require.NoError(t, m.ensureDirectories(id)) stored := &StoredMetadata{ Id: id, + Name: "surviving metadata", GPUFramework: devices.VGPUFrameworkVendorVFIO, GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", } @@ -131,61 +130,10 @@ func TestCleanupFailedCreateRewritesFullMetadataWhenDirectoryIsReadOnly(t *testi assert.True(t, persistTestVGPURetention(m, context.Background(), id, stored)) retained, err := m.loadMetadata(id) require.NoError(t, err) + assert.Equal(t, stored.Name, retained.Name) assert.Equal(t, stored.GPUFramework, retained.GPUFramework) assert.Equal(t, stored.GPUDevicePath, retained.GPUDevicePath) - assert.True(t, retained.GPURetainedForCleanup) - - _, err = m.startInstance(context.Background(), id, StartInstanceRequest{}) - assert.ErrorIs(t, err, errVGPURetentionStub) -} - -func TestVGPURetentionWrapPending(t *testing.T) { - cause := errors.New("boot failed") - - retention := vgpuRetention{instanceID: "inst-1"} - assert.Same(t, cause, retention.wrapPending(cause)) - - retention.retained = true - pending := retention.wrapPending(cause) - var cleanupPending *VGPUCleanupPendingError - require.ErrorAs(t, pending, &cleanupPending) - assert.False(t, cleanupPending.Retained) - - retention.persisted = true - pending = retention.wrapPending(cause) - require.ErrorAs(t, pending, &cleanupPending) - assert.True(t, cleanupPending.Retained) -} - -func TestVGPUDevicePendingCleanup(t *testing.T) { - t.Parallel() - - device := devices.VGPUDevice{ - Framework: devices.VGPUFrameworkVendorVFIO, - SysfsPath: "/sys/bus/pci/devices/0000:82:00.4", - } - cause := errors.New("rollback failed") - pending := &devices.VGPUCreateCleanupPendingError{Device: device, Err: cause} - - wrapped := fmt.Errorf("create failed: %w", pending) - actual, ok := vgpuDevicePendingCleanup(wrapped) - require.True(t, ok) - assert.Equal(t, device, *actual) - - assignedAt := time.Now().UTC() - retained := retainedVGPUFromCreateErrorForTest(StoredMetadata{Id: "inst-1", Name: "named", Image: "img"}, assignedAt, wrapped) - require.NotNil(t, retained) - assert.Equal(t, "inst-1", retained.Id) - assert.Equal(t, "named", retained.Name, "identity fields must survive into the retention stub") - assert.Equal(t, "img", retained.Image) - assert.Equal(t, device.Framework, retained.GPUFramework) - assert.Equal(t, device.SysfsPath, retained.GPUDevicePath) - assert.Equal(t, assignedAt, *retained.GPUAssignedAt) - - actual, ok = vgpuDevicePendingCleanup(cause) - assert.False(t, ok) - assert.Nil(t, actual) - assert.Nil(t, retainedVGPUFromCreateErrorForTest(StoredMetadata{Id: "inst-1"}, assignedAt, cause)) + assert.False(t, retained.GPURetainedForCleanup) } type startRetentionNetworkManager struct { @@ -333,24 +281,6 @@ func TestStartReportsUnretainedVGPUWhenRetentionSaveFails(t *testing.T) { assert.Empty(t, stored.GPUDevicePath, "retention save failed, so no assignment should be recorded") } -func TestStartDoesNotRestrictVGPUHypervisor(t *testing.T) { - m, id := newStartRollbackVGPUManager(t, func(context.Context, devices.VGPUAssignment) error { - return nil - }) - meta, err := m.loadMetadata(id) - require.NoError(t, err) - meta.HypervisorType = hypervisor.TypeCloudHypervisor - require.NoError(t, m.saveMetadata(meta)) - - cause := errors.New("create failed") - m.createVGPU = func(context.Context, string, string) (*devices.VGPUDevice, error) { - return nil, cause - } - - _, err = m.startInstance(context.Background(), id, StartInstanceRequest{}) - assert.ErrorIs(t, err, cause) -} - func TestStartRollbackClearsVGPUAssignmentAfterSuccessfulDestroy(t *testing.T) { var destroyed []devices.VGPUAssignment m, id := newStartRollbackVGPUManager(t, func(_ context.Context, assignment devices.VGPUAssignment) error { @@ -439,37 +369,24 @@ func TestCleanupStartVGPUReportsRetainedWhenMidStartSaveSurvives(t *testing.T) { func TestCleanupStartVGPURestoresMetadataAfterBootFailure(t *testing.T) { m := &manager{ - paths: paths.New(t.TempDir()), - destroyVGPU: func(context.Context, devices.VGPUAssignment) error { - return nil - }, + paths: paths.New(t.TempDir()), + destroyVGPU: func(context.Context, devices.VGPUAssignment) error { return nil }, } const id = "failed-start" require.NoError(t, m.ensureDirectories(id)) previousStart := time.Now().Add(-time.Hour).UTC() - previousProgramStart := previousStart.Add(time.Second) exitCode := 1 rollbackMeta := metadata{StoredMetadata: StoredMetadata{ - Id: id, - Name: "original name", - GPUProfile: "NVIDIA L40S-2Q", - Entrypoint: []string{"old-entrypoint"}, - Cmd: []string{"old-command"}, - StartedAt: &previousStart, - ProgramStartedAt: &previousProgramStart, - ExitCode: &exitCode, - ExitMessage: "previous exit", + Id: id, + Name: "original name", + GPUProfile: "NVIDIA L40S-2Q", + Entrypoint: []string{"old-entrypoint"}, + StartedAt: &previousStart, + ExitCode: &exitCode, + ExitMessage: "previous exit", }} - - partial := rollbackMeta - partial.Name = "concurrent update" - partial.Entrypoint = []string{"new-entrypoint"} - partial.Cmd = []string{"new-command"} - partial.StartedAt = ptr(time.Now().UTC()) - partial.ProgramStartedAt = nil - partial.ExitCode = nil - partial.ExitMessage = "" + partial := metadata{StoredMetadata: StoredMetadata{Id: id, Name: "partial start"}} assignedAt := time.Now().UTC() device := &devices.VGPUDevice{ Framework: devices.VGPUFrameworkVendorVFIO, @@ -482,15 +399,7 @@ func TestCleanupStartVGPURestoresMetadataAfterBootFailure(t *testing.T) { stored, err := m.loadMetadata(id) require.NoError(t, err) - assert.Equal(t, "concurrent update", stored.Name) - assert.Equal(t, rollbackMeta.Entrypoint, stored.Entrypoint) - assert.Equal(t, rollbackMeta.Cmd, stored.Cmd) - assert.Equal(t, rollbackMeta.StartedAt, stored.StartedAt) - assert.Equal(t, rollbackMeta.ProgramStartedAt, stored.ProgramStartedAt) - assert.Equal(t, rollbackMeta.ExitCode, stored.ExitCode) - assert.Equal(t, rollbackMeta.ExitMessage, stored.ExitMessage) - assert.Empty(t, stored.GPUDevicePath) - assert.Nil(t, stored.GPUAssignedAt) + assert.Equal(t, rollbackMeta.StoredMetadata, stored.StoredMetadata) } func TestVGPUAssignmentClaimedByLiveInstanceFailsOnInvalidMetadata(t *testing.T) { @@ -500,7 +409,7 @@ func TestVGPUAssignmentClaimedByLiveInstanceFailsOnInvalidMetadata(t *testing.T) require.NoError(t, m.ensureDirectories("invalid-instance")) require.NoError(t, os.WriteFile(m.paths.InstanceMetadata("invalid-instance"), []byte("{"), 0o644)) - _, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "other-instance", "/sys/bus/pci/devices/0000:82:00.4") + _, err := m.vgpuAssignmentClaimedByLiveInstance("other-instance", "/sys/bus/pci/devices/0000:82:00.4") require.Error(t, err) } @@ -521,7 +430,7 @@ func TestVGPUAssignmentClaimedByLiveInstanceNormalizesLegacyMdevPath(t *testing. }, }})) - claimed, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "other-instance", "/sys/bus/mdev/devices/legacy-uuid") + claimed, err := m.vgpuAssignmentClaimedByLiveInstance("other-instance", "/sys/bus/mdev/devices/legacy-uuid") require.NoError(t, err) assert.True(t, claimed) } @@ -533,7 +442,7 @@ func TestVGPUAssignmentClaimedByLiveInstanceLiveness(t *testing.T) { deadPID := 1 << 30 require.False(t, ProcessExists(deadPID)) recent := time.Now().UTC() - stale := recent.Add(-VGPUAssignmentStartupGracePeriod - time.Minute) + stale := recent.Add(-devices.VGPUAssignmentGracePeriod - time.Minute) tests := []struct { name string assignedAt *time.Time @@ -559,7 +468,7 @@ func TestVGPUAssignmentClaimedByLiveInstanceLiveness(t *testing.T) { HypervisorProcessIdentity: HypervisorProcessIdentity{HypervisorPID: tt.pid}, }})) - claimed, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "requester", devicePath) + claimed, err := m.vgpuAssignmentClaimedByLiveInstance("requester", devicePath) if tt.wantErr != "" { require.ErrorContains(t, err, tt.wantErr) return @@ -635,37 +544,3 @@ func TestStoredVGPUDevicePath(t *testing.T) { })) assert.Empty(t, storedVGPUDevicePath(&StoredMetadata{})) } - -func TestReleaseStoredVGPURetainsMetadataOnFailure(t *testing.T) { - t.Parallel() - - m := &manager{paths: paths.New(t.TempDir())} - stored := &StoredMetadata{ - GPUFramework: devices.VGPUFramework("future-framework"), - GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", - } - err := m.releaseStoredVGPU(context.Background(), stored) - assert.Error(t, err) - assert.Equal(t, devices.VGPUFramework("future-framework"), stored.GPUFramework) - assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", stored.GPUDevicePath) -} - -func TestSetAndClearStoredVGPUDevice(t *testing.T) { - t.Parallel() - - assignedAt := time.Now().UTC() - stored := &StoredMetadata{} - setStoredVGPUDevice(stored, &devices.VGPUDevice{ - Framework: devices.VGPUFrameworkVendorVFIO, - SysfsPath: "/sys/bus/pci/devices/0000:82:00.4", - }, assignedAt) - assert.Equal(t, devices.VGPUFrameworkVendorVFIO, stored.GPUFramework) - assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", stored.GPUDevicePath) - assert.Equal(t, assignedAt, *stored.GPUAssignedAt) - - clearStoredVGPUDevice(stored) - assert.Empty(t, stored.GPUFramework) - assert.Empty(t, stored.GPUDevicePath) - assert.Empty(t, stored.GPUMdevUUID) - assert.Nil(t, stored.GPUAssignedAt) -} From be458d17f21e06fd66e62c3338ab51dfea05df30 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Thu, 27 Aug 2026 14:11:57 +0000 Subject: [PATCH 73/76] Simplify vGPU retention persistence to a single attempt --- lib/instances/vgpu.go | 5 +++-- lib/instances/vgpu_retention.go | 19 +++---------------- lib/instances/vgpu_test.go | 33 ++------------------------------- 3 files changed, 8 insertions(+), 49 deletions(-) diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index 7b70e807c..6d45d82f6 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -94,8 +94,7 @@ func (m *manager) cleanupStartVGPU(ctx context.Context, instanceID string, devic if !retained { return false, false } - meta, loadErr := m.loadMetadata(instanceID) - return true, loadErr == nil && storedVGPUDevicePath(&meta.StoredMetadata) == device.SysfsPath + return true, false } return retained, retained } @@ -132,6 +131,8 @@ func (m *manager) releaseStoredVGPU(ctx context.Context, stored *StoredMetadata) } func (m *manager) vgpuAssignmentClaimedByLiveInstance(excludeID, devicePath string) (bool, error) { + // Each vendor VFIO release lists and loads every instance's metadata. This is + // acceptable at GPU-host scale (tens of VFs and instances); revisit at hundreds. allMetadata, err := m.listMetadataForReconcile() if err != nil { return false, fmt.Errorf("list instances for vGPU release check: %w", err) diff --git a/lib/instances/vgpu_retention.go b/lib/instances/vgpu_retention.go index 2804e8abf..a0e048bdc 100644 --- a/lib/instances/vgpu_retention.go +++ b/lib/instances/vgpu_retention.go @@ -59,32 +59,19 @@ func (m *manager) persistVGPURetention(ctx context.Context, retention *vgpuReten defer func() { m.recordVGPURetainedAssignment(ctx, vgpuRetentionOperationCreate, retention.persisted) }() - retentionSurvives := func() bool { - meta, err := m.loadMetadata(id) - if err == nil && storedVGPUDevicePath(&meta.StoredMetadata) != "" { - if err := m.saveVGPURetentionStub(retainedVGPU); err != nil { - log.ErrorContext(ctx, "failed to replace surviving instance metadata with vGPU retention stub; preserving existing assignment claim", "instance_id", id, "error", err) - } - return true - } - if err := m.deleteInstanceData(id); err != nil { - log.ErrorContext(ctx, "failed to delete stale instance data after retention failure", "instance_id", id, "error", err) - } - return false - } + + // An unpersisted retention leaves no metadata claim. The periodic reconciler releases + // the VF after its grace period once no open VFIO handles remain. if err := m.deleteInstanceData(id); err != nil { log.ErrorContext(ctx, "failed to clean instance data before retaining vGPU assignment", "instance_id", id, "error", err) - retention.persisted = retentionSurvives() return } if err := m.ensureDirectories(id); err != nil { log.ErrorContext(ctx, "failed to retain instance data after vGPU cleanup failure", "instance_id", id, "error", err) - retention.persisted = retentionSurvives() return } if err := m.saveVGPURetentionStub(retainedVGPU); err != nil { log.ErrorContext(ctx, "failed to retain vGPU assignment metadata after cleanup failure", "instance_id", id, "error", err) - retention.persisted = retentionSurvives() return } retention.persisted = true diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index b575c7546..d8814e17e 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -107,35 +107,6 @@ func TestCleanupFailedCreateReportsUnpersistedRetention(t *testing.T) { require.Error(t, err, "the lost retention leaves no metadata claim, so the periodic sweep releases the VF") } -func TestCleanupFailedCreatePreservesClaimWhenStubSaveFails(t *testing.T) { - if os.Geteuid() == 0 { - t.Skip("root bypasses directory permissions") - } - - m := &manager{paths: paths.New(t.TempDir())} - const id = "failed-create" - require.NoError(t, m.ensureDirectories(id)) - stored := &StoredMetadata{ - Id: id, - Name: "surviving metadata", - GPUFramework: devices.VGPUFrameworkVendorVFIO, - GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", - } - require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: *stored})) - - instanceDir := filepath.Dir(m.paths.InstanceMetadata(id)) - require.NoError(t, os.Chmod(instanceDir, 0o555)) - t.Cleanup(func() { _ = os.Chmod(instanceDir, 0o755) }) - - assert.True(t, persistTestVGPURetention(m, context.Background(), id, stored)) - retained, err := m.loadMetadata(id) - require.NoError(t, err) - assert.Equal(t, stored.Name, retained.Name) - assert.Equal(t, stored.GPUFramework, retained.GPUFramework) - assert.Equal(t, stored.GPUDevicePath, retained.GPUDevicePath) - assert.False(t, retained.GPURetainedForCleanup) -} - type startRetentionNetworkManager struct { network.Manager config network.NetworkConfig @@ -338,7 +309,7 @@ func TestStartRollbackRetainsVGPUAssignmentAfterFailedDestroy(t *testing.T) { assert.Empty(t, stored.Entrypoint) } -func TestCleanupStartVGPUReportsRetainedWhenMidStartSaveSurvives(t *testing.T) { +func TestCleanupStartVGPUReportsUnpersistedRetentionWhenRollbackSaveFails(t *testing.T) { if os.Geteuid() == 0 { t.Skip("root bypasses directory permissions") } @@ -364,7 +335,7 @@ func TestCleanupStartVGPUReportsRetainedWhenMidStartSaveSurvives(t *testing.T) { retained, persisted := m.cleanupStartVGPU(context.Background(), id, &device, assignedAt, rollbackMeta) assert.True(t, retained) - assert.True(t, persisted, "a surviving mid-start save keeps the assignment recoverable via delete") + assert.False(t, persisted) } func TestCleanupStartVGPURestoresMetadataAfterBootFailure(t *testing.T) { From a43dee1d1b61d35d54fe3882b883639d9d980566 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Thu, 27 Aug 2026 18:52:33 +0000 Subject: [PATCH 74/76] Report vGPU retention as persisted when the assignment claim survives When rollback release fails and the subsequent retention save also fails, the assignment save from earlier in start may still hold the claim on disk. Check for that surviving claim before reporting the retention as unpersisted, so the API does not emit vgpu_unretained_instance for an assignment that is still durably attributed. --- lib/instances/vgpu.go | 7 +++++ lib/instances/vgpu_test.go | 52 ++++++++++++++++++++++++-------------- 2 files changed, 40 insertions(+), 19 deletions(-) diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index 6d45d82f6..00e32ba01 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -94,6 +94,13 @@ func (m *manager) cleanupStartVGPU(ctx context.Context, instanceID string, devic if !retained { return false, false } + // The pre-cleanup assignment save may already hold this claim on disk, in + // which case the retention is durable despite the failed rollback save. + if onDisk, loadErr := m.loadMetadata(instanceID); loadErr == nil && + onDisk.GPUDevicePath == device.SysfsPath && + onDisk.GPUAssignedAt != nil && onDisk.GPUAssignedAt.Equal(assignedAt) { + return true, true + } return true, false } return retained, retained diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index d8814e17e..e82773115 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -309,33 +309,47 @@ func TestStartRollbackRetainsVGPUAssignmentAfterFailedDestroy(t *testing.T) { assert.Empty(t, stored.Entrypoint) } -func TestCleanupStartVGPUReportsUnpersistedRetentionWhenRollbackSaveFails(t *testing.T) { +func TestCleanupStartVGPUReportsRetentionWhenRollbackSaveFails(t *testing.T) { if os.Geteuid() == 0 { t.Skip("root bypasses directory permissions") } - m, id := newStartRollbackVGPUManager(t, func(context.Context, devices.VGPUAssignment) error { - return errors.New("destroy failed") - }) - device := devices.VGPUDevice{ - Framework: devices.VGPUFrameworkVendorVFIO, - SysfsPath: "/sys/bus/pci/devices/0000:82:00.4", + tests := []struct { + name string + assignmentSaved bool + wantPersisted bool + }{ + {name: "assignment save survived", assignmentSaved: true, wantPersisted: true}, + {name: "assignment never saved", assignmentSaved: false, wantPersisted: false}, } - assignedAt := time.Now().UTC() + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + m, id := newStartRollbackVGPUManager(t, func(context.Context, devices.VGPUAssignment) error { + return errors.New("destroy failed") + }) + device := devices.VGPUDevice{ + Framework: devices.VGPUFrameworkVendorVFIO, + SysfsPath: "/sys/bus/pci/devices/0000:82:00.4", + } + assignedAt := time.Now().UTC() - meta, err := m.loadMetadata(id) - require.NoError(t, err) - rollbackMeta := *meta - setStoredVGPUDevice(&meta.StoredMetadata, &device, assignedAt) - require.NoError(t, m.saveMetadata(meta)) + meta, err := m.loadMetadata(id) + require.NoError(t, err) + rollbackMeta := *meta + if tt.assignmentSaved { + setStoredVGPUDevice(&meta.StoredMetadata, &device, assignedAt) + require.NoError(t, m.saveMetadata(meta)) + } - instanceDir := filepath.Dir(m.paths.InstanceMetadata(id)) - require.NoError(t, os.Chmod(instanceDir, 0o555)) - t.Cleanup(func() { _ = os.Chmod(instanceDir, 0o755) }) + instanceDir := filepath.Dir(m.paths.InstanceMetadata(id)) + require.NoError(t, os.Chmod(instanceDir, 0o555)) + t.Cleanup(func() { _ = os.Chmod(instanceDir, 0o755) }) - retained, persisted := m.cleanupStartVGPU(context.Background(), id, &device, assignedAt, rollbackMeta) - assert.True(t, retained) - assert.False(t, persisted) + retained, persisted := m.cleanupStartVGPU(context.Background(), id, &device, assignedAt, rollbackMeta) + assert.True(t, retained) + assert.Equal(t, tt.wantPersisted, persisted) + }) + } } func TestCleanupStartVGPURestoresMetadataAfterBootFailure(t *testing.T) { From 07da77245f8a66efbd2d0fa88c4e800885c11fc8 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Fri, 28 Aug 2026 13:49:27 +0000 Subject: [PATCH 75/76] Protect claimed mdevs during reconciliation --- lib/devices/mdev_darwin.go | 2 +- lib/devices/mdev_linux.go | 22 +++++++++++++++++++--- lib/devices/vgpu_linux.go | 22 +++++++++++++++++----- lib/devices/vgpu_linux_test.go | 24 ++++++++++++++++++++++++ lib/instances/vgpu_reconcile.go | 13 ++++++------- lib/instances/vgpu_reconcile_test.go | 20 +++++++++++--------- 6 files changed, 78 insertions(+), 25 deletions(-) diff --git a/lib/devices/mdev_darwin.go b/lib/devices/mdev_darwin.go index 4b726bb08..93d3adbf7 100644 --- a/lib/devices/mdev_darwin.go +++ b/lib/devices/mdev_darwin.go @@ -58,7 +58,7 @@ func DestroyVGPU(ctx context.Context, assignment VGPUAssignment) error { return nil } -func ReconcileVGPUs(ctx context.Context, protectedDevicePaths map[string]struct{}, sweepVendorVFIO bool) error { +func ReconcileVGPUs(ctx context.Context, protectedDevicePaths map[string]struct{}, sweepDevices bool) error { return nil } diff --git a/lib/devices/mdev_linux.go b/lib/devices/mdev_linux.go index 0908e8765..423d8473d 100644 --- a/lib/devices/mdev_linux.go +++ b/lib/devices/mdev_linux.go @@ -707,17 +707,27 @@ func mdevPastGracePeriod(mdevUUID string, gracePeriod time.Duration) (bool, time return age >= gracePeriod, age, nil } +func protectedMdevUUIDs(instanceInfos []MdevReconcileInfo) map[string]struct{} { + protected := make(map[string]struct{}, len(instanceInfos)) + for _, info := range instanceInfos { + if info.MdevUUID != "" && info.IsRunning { + protected[info.MdevUUID] = struct{}{} + } + } + return protected +} + // ReconcileMdevs destroys orphaned mdevs on managed VFs. -// This is called on server startup to clean up stale mdevs from previous runs. // // Policy: // - Consider only mdevs whose parent VF is currently managed by hypeman (discoverable via /sys/class/mdev_bus) +// - Keep mdevs claimed by live instance metadata // - Keep mdevs whose VFIO group has an open file handle (/dev/vfio/) // - Keep mdevs younger than a short grace period to avoid racing very recent state transitions // - Delete all remaining mdevs func ReconcileMdevs(ctx context.Context, instanceInfos []MdevReconcileInfo) error { log := logger.FromContext(ctx) - _ = instanceInfos + protectedMdevs := protectedMdevUUIDs(instanceInfos) vfs, err := discoverMdevVFs() if err != nil { @@ -750,13 +760,18 @@ func ReconcileMdevs(ctx context.Context, instanceInfos []MdevReconcileInfo) erro ) groupInUseCache := make(map[int]bool) - var destroyed, failedDestroy, skippedUnmanagedVF, skippedInUse, skippedGrace, skippedProbeError int + var destroyed, failedDestroy, skippedUnmanagedVF, skippedClaimed, skippedInUse, skippedGrace, skippedProbeError int for _, mdev := range mdevs { if _, ok := managedVFs[mdev.VFAddress]; !ok { log.DebugContext(ctx, "skipping mdev on unmanaged VF", "uuid", mdev.UUID, "vf", mdev.VFAddress) skippedUnmanagedVF++ continue } + if _, ok := protectedMdevs[mdev.UUID]; ok { + log.DebugContext(ctx, "skipping mdev claimed by live instance", "uuid", mdev.UUID) + skippedClaimed++ + continue + } group, err := mdevIOMMUGroup(mdev.UUID) if err != nil { @@ -817,6 +832,7 @@ func ReconcileMdevs(ctx context.Context, instanceInfos []MdevReconcileInfo) erro "destroyed", destroyed, "failed_destroy", failedDestroy, "skipped_unmanaged_vf", skippedUnmanagedVF, + "skipped_claimed", skippedClaimed, "skipped_in_use", skippedInUse, "skipped_grace", skippedGrace, "skipped_probe_error", skippedProbeError, diff --git a/lib/devices/vgpu_linux.go b/lib/devices/vgpu_linux.go index 36bf86c9e..354217fd5 100644 --- a/lib/devices/vgpu_linux.go +++ b/lib/devices/vgpu_linux.go @@ -104,20 +104,32 @@ func DestroyVGPU(ctx context.Context, assignment VGPUAssignment) error { } } +func mdevReconcileInfos(protectedDevicePaths map[string]struct{}) []MdevReconcileInfo { + instanceInfos := make([]MdevReconcileInfo, 0, len(protectedDevicePaths)) + for devicePath := range protectedDevicePaths { + instanceInfos = append(instanceInfos, MdevReconcileInfo{ + MdevUUID: filepath.Base(devicePath), + IsRunning: true, + }) + } + return instanceInfos +} + // ReconcileVGPUs releases orphaned vGPU assignments. -func ReconcileVGPUs(ctx context.Context, protectedDevicePaths map[string]struct{}, sweepVendorVFIO bool) error { +func ReconcileVGPUs(ctx context.Context, protectedDevicePaths map[string]struct{}, sweepDevices bool) error { framework, _, err := DiscoverVGPU() if err != nil { return err } + if !sweepDevices { + return nil + } switch framework { case VGPUFrameworkMdev: - return ReconcileMdevs(ctx, nil) + return ReconcileMdevs(ctx, mdevReconcileInfos(protectedDevicePaths)) case VGPUFrameworkVendorVFIO: - if sweepVendorVFIO { - return hostVendorVFIO.reconcile(ctx, protectedDevicePaths) - } + return hostVendorVFIO.reconcile(ctx, protectedDevicePaths) } return nil } diff --git a/lib/devices/vgpu_linux_test.go b/lib/devices/vgpu_linux_test.go index 7b03d9c26..5bb52da1d 100644 --- a/lib/devices/vgpu_linux_test.go +++ b/lib/devices/vgpu_linux_test.go @@ -12,6 +12,30 @@ import ( "github.com/stretchr/testify/require" ) +func TestMdevReconcileInfosProtectClaimedDevicePath(t *testing.T) { + t.Parallel() + + infos := mdevReconcileInfos(map[string]struct{}{ + "/sys/bus/mdev/devices/claimed": {}, + }) + + assert.Equal(t, []MdevReconcileInfo{{MdevUUID: "claimed", IsRunning: true}}, infos) +} + +func TestProtectedMdevUUIDs(t *testing.T) { + t.Parallel() + + protected := protectedMdevUUIDs([]MdevReconcileInfo{ + {MdevUUID: "claimed", IsRunning: true}, + {MdevUUID: "stale"}, + {IsRunning: true}, + }) + + assert.Contains(t, protected, "claimed") + assert.NotContains(t, protected, "stale") + assert.Len(t, protected, 1) +} + func TestDiscoverVGPUWithPropagatesMdevError(t *testing.T) { t.Parallel() diff --git a/lib/instances/vgpu_reconcile.go b/lib/instances/vgpu_reconcile.go index 066c9bdee..27ecb9e01 100644 --- a/lib/instances/vgpu_reconcile.go +++ b/lib/instances/vgpu_reconcile.go @@ -54,17 +54,17 @@ func (m *manager) StartVGPUReconciler(ctx context.Context) { func (m *manager) ReconcileVGPUs(ctx context.Context) { log := logger.FromContext(ctx) protected, err := m.reconcileVGPUAssignments(ctx) - sweepVendorVFIO := err == nil + sweepDevices := err == nil if err != nil { m.recordVGPUReconcileFailure(ctx, vgpuReconcileStageListInstances) - log.ErrorContext(ctx, "failed to list instances for vGPU reconcile protection; skipping vendor VFIO sweep until the next pass, mdev reconcile still runs", "error", err) + log.ErrorContext(ctx, "failed to list instances for vGPU reconcile protection; skipping device sweep until the next pass", "error", err) protected = make(map[string]struct{}) } reconcileDevices := m.reconcileVGPUDevices if reconcileDevices == nil { reconcileDevices = devices.ReconcileVGPUs } - if err := reconcileDevices(ctx, protected, sweepVendorVFIO); err != nil { + if err := reconcileDevices(ctx, protected, sweepDevices); err != nil { m.recordVGPUReconcileFailure(ctx, vgpuReconcileStageReconcileDevices) log.WarnContext(ctx, "failed to reconcile vGPU devices", "error", err) } @@ -82,14 +82,13 @@ func (m *manager) reconcileVGPUAssignments(ctx context.Context) (map[string]stru protected := make(map[string]struct{}) for i := range allMetadata { stored := &allMetadata[i] - if storedVGPUDevicePath(stored) == "" { + devicePath := storedVGPUDevicePath(stored) + if devicePath == "" { continue } hypervisorLive := hypervisorMayBeAlive(stored.HypervisorProcessIdentity, stored.SocketPath) if vgpuAssignmentMayBeLive(stored, m.nowUTC(), hypervisorLive) { - if stored.GPUDevicePath != "" { - protected[stored.GPUDevicePath] = struct{}{} - } + protected[devicePath] = struct{}{} continue } m.releaseStaleVGPUAssignment(ctx, stored.Id) diff --git a/lib/instances/vgpu_reconcile_test.go b/lib/instances/vgpu_reconcile_test.go index 7a9b66731..8b8d71b66 100644 --- a/lib/instances/vgpu_reconcile_test.go +++ b/lib/instances/vgpu_reconcile_test.go @@ -33,9 +33,9 @@ func TestReconcileVGPUsBoundsStartupProtection(t *testing.T) { destroyed = append(destroyed, assignment) return nil }, - reconcileVGPUDevices: func(_ context.Context, p map[string]struct{}, sweepVendorVFIO bool) error { + reconcileVGPUDevices: func(_ context.Context, p map[string]struct{}, sweepDevices bool) error { protected = p - assert.True(t, sweepVendorVFIO) + assert.True(t, sweepDevices) return nil }, } @@ -45,6 +45,7 @@ func TestReconcileVGPUsBoundsStartupProtection(t *testing.T) { {Id: "legacy", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.6"}, {Id: "dead", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.7", HypervisorProcessIdentity: HypervisorProcessIdentity{HypervisorPID: &deadPID}}, {Id: "stale-pid-booting", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.8", HypervisorProcessIdentity: HypervisorProcessIdentity{HypervisorPID: &deadPID}, GPUAssignedAt: &recent}, + {Id: "legacy-mdev-booting", GPUMdevUUID: "test-mdev", GPUAssignedAt: &recent}, } for i := range instances { require.NoError(t, m.ensureDirectories(instances[i].Id)) @@ -57,6 +58,7 @@ func TestReconcileVGPUsBoundsStartupProtection(t *testing.T) { assert.NotContains(t, protected, "/sys/bus/pci/devices/0000:82:00.6") assert.NotContains(t, protected, "/sys/bus/pci/devices/0000:82:00.7") assert.Contains(t, protected, "/sys/bus/pci/devices/0000:82:00.8") + assert.Contains(t, protected, "/sys/bus/mdev/devices/test-mdev") for _, id := range []string{"orphaned", "legacy", "dead"} { stored, err := m.loadMetadata(id) @@ -71,14 +73,14 @@ func TestReconcileVGPUsBoundsStartupProtection(t *testing.T) { orphaned, err := m.loadMetadata("orphaned") require.NoError(t, err) assert.Equal(t, "NVIDIA L40S-2Q", orphaned.GPUProfile) - for _, id := range []string{"booting", "stale-pid-booting"} { + for _, id := range []string{"booting", "stale-pid-booting", "legacy-mdev-booting"} { stored, err := m.loadMetadata(id) require.NoError(t, err) - assert.NotEmpty(t, stored.GPUDevicePath, "live assignment on %s must be kept", id) + assert.NotEmpty(t, storedVGPUDevicePath(&stored.StoredMetadata), "live assignment on %s must be kept", id) } } -func TestReconcileVGPUsSkipsVendorSweepWhenListingFails(t *testing.T) { +func TestReconcileVGPUsSkipsDeviceSweepWhenListingFails(t *testing.T) { if os.Geteuid() == 0 { t.Skip("root bypasses directory permissions") } @@ -86,8 +88,8 @@ func TestReconcileVGPUsSkipsVendorSweepWhenListingFails(t *testing.T) { var sweeps []bool m := &manager{ paths: paths.New(t.TempDir()), - reconcileVGPUDevices: func(_ context.Context, _ map[string]struct{}, sweepVendorVFIO bool) error { - sweeps = append(sweeps, sweepVendorVFIO) + reconcileVGPUDevices: func(_ context.Context, _ map[string]struct{}, sweepDevices bool) error { + sweeps = append(sweeps, sweepDevices) return nil }, } @@ -100,11 +102,11 @@ func TestReconcileVGPUsSkipsVendorSweepWhenListingFails(t *testing.T) { m.ReconcileVGPUs(t.Context()) require.Equal(t, []bool{false}, sweeps, - "a listing failure must skip the vendor sweep, not run it with an empty protection set") + "a listing failure must skip the device sweep, not run it with an empty protection set") require.NoError(t, os.Chmod(instanceDir, 0o755)) m.ReconcileVGPUs(t.Context()) - assert.Equal(t, []bool{false, true}, sweeps, "the next pass retries the vendor sweep") + assert.Equal(t, []bool{false, true}, sweeps, "the next pass retries the device sweep") } func TestReconcileVGPUsKeepsAssignmentWhenReleaseFails(t *testing.T) { From 4f13a3c174a3a2089212fbf94e7c054cd99784ab Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Fri, 28 Aug 2026 15:56:07 +0000 Subject: [PATCH 76/76] Unify vGPU retention wrapping and dedupe test fixtures --- lib/instances/create.go | 3 ++- lib/instances/start.go | 19 +++++++++--------- lib/instances/vgpu_retention.go | 5 ----- lib/instances/vgpu_test.go | 35 +++++++++++++-------------------- 4 files changed, 26 insertions(+), 36 deletions(-) diff --git a/lib/instances/create.go b/lib/instances/create.go index 525f2c262..cd50deb5c 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -280,7 +280,8 @@ func (m *manager) createInstance( var gpuAssignedAt *time.Time retention := vgpuRetention{instanceID: id} - defer retention.deferWrapPending(&retErr) + // Deferred before cu.Clean so rollback records retention before this wraps the error. + defer func() { retErr = retention.wrapPending(retErr) }() cu := cleanup.Make(func() { log.DebugContext(ctx, "cleaning up instance on error", "instance_id", id) m.persistVGPURetention(ctx, &retention) diff --git a/lib/instances/start.go b/lib/instances/start.go index ea8c2a6f4..37b836427 100644 --- a/lib/instances/start.go +++ b/lib/instances/start.go @@ -121,7 +121,8 @@ func (m *manager) startInstance( // Setup cleanup stack for automatic rollback on errors retention := vgpuRetention{instanceID: id} - defer retention.deferWrapPending(&retErr) + // Deferred before cu.Clean so rollback records retention before this wraps the error. + defer func() { retErr = retention.wrapPending(retErr) }() cu := cleanup.Make(func() {}) defer cu.Clean() @@ -177,20 +178,20 @@ func (m *manager) startInstance( device, err := m.createVGPUDevice(ctx, stored.GPUProfile, id) if err != nil { log.ErrorContext(ctx, "failed to create vGPU", "instance_id", id, "profile", stored.GPUProfile, "error", err) + wrapped := fmt.Errorf("create vGPU for profile %s: %w", stored.GPUProfile, err) if pendingDevice, ok := vgpuDevicePendingCleanup(err); ok { - assignedAt := m.nowUTC() retentionMeta := rollbackMeta - setStoredVGPUDevice(&retentionMeta.StoredMetadata, pendingDevice, assignedAt) - wrapped := fmt.Errorf("create vGPU for profile %s: %w", stored.GPUProfile, err) + setStoredVGPUDevice(&retentionMeta.StoredMetadata, pendingDevice, m.nowUTC()) + persisted := true if saveErr := m.saveMetadata(&retentionMeta); saveErr != nil { - m.recordVGPURetainedAssignment(ctx, vgpuRetentionOperationStart, false) log.ErrorContext(ctx, "failed to retain vGPU assignment after create rollback failure", "instance_id", id, "error", saveErr) - return nil, &VGPUCleanupPendingError{InstanceID: id, Err: fmt.Errorf("%w; retain assignment: %v", wrapped, saveErr)} + wrapped = fmt.Errorf("%w; retain assignment: %v", wrapped, saveErr) + persisted = false } - m.recordVGPURetainedAssignment(ctx, vgpuRetentionOperationStart, true) - return nil, &VGPUCleanupPendingError{InstanceID: id, Retained: true, Err: wrapped} + retention.markRetained(persisted) + m.recordVGPURetainedAssignment(ctx, vgpuRetentionOperationStart, persisted) } - return nil, fmt.Errorf("create vGPU for profile %s: %w", stored.GPUProfile, err) + return nil, wrapped } assignedAt := m.nowUTC() setStoredVGPUDevice(stored, device, assignedAt) diff --git a/lib/instances/vgpu_retention.go b/lib/instances/vgpu_retention.go index a0e048bdc..1bcba2c42 100644 --- a/lib/instances/vgpu_retention.go +++ b/lib/instances/vgpu_retention.go @@ -42,11 +42,6 @@ func (r *vgpuRetention) wrapPending(err error) error { return &VGPUCleanupPendingError{InstanceID: r.instanceID, Retained: r.persisted, Err: err} } -// Defer before cleanup so rollback records retention before this wraps the error. -func (r *vgpuRetention) deferWrapPending(retErr *error) { - *retErr = r.wrapPending(*retErr) -} - func (m *manager) persistVGPURetention(ctx context.Context, retention *vgpuRetention) { if retention.stub == nil { m.deleteInstanceData(retention.instanceID) diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index e82773115..07cdcb8e5 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -18,6 +18,16 @@ import ( "github.com/stretchr/testify/require" ) +func testVendorVFIODevice(profileName string) devices.VGPUDevice { + return devices.VGPUDevice{ + Framework: devices.VGPUFrameworkVendorVFIO, + VFAddress: "0000:82:00.4", + ProfileType: "1148", + ProfileName: profileName, + SysfsPath: "/sys/bus/pci/devices/0000:82:00.4", + } +} + func persistTestVGPURetention(m *manager, ctx context.Context, id string, stub *StoredMetadata) bool { retention := vgpuRetention{instanceID: id, stub: stub, retained: stub != nil} m.persistVGPURetention(ctx, &retention) @@ -131,13 +141,8 @@ func newStartRollbackVGPUManager(t *testing.T, destroy func(context.Context, dev instanceLocks: sync.Map{}, bootMarkerScans: sync.Map{}, createVGPU: func(_ context.Context, profileName, _ string) (*devices.VGPUDevice, error) { - return &devices.VGPUDevice{ - Framework: devices.VGPUFrameworkVendorVFIO, - VFAddress: "0000:82:00.4", - ProfileType: "1148", - ProfileName: profileName, - SysfsPath: "/sys/bus/pci/devices/0000:82:00.4", - }, nil + device := testVendorVFIODevice(profileName) + return &device, nil }, destroyVGPU: destroy, } @@ -180,13 +185,7 @@ func TestStartRetainsVGPUWhenCreateRollbackFails(t *testing.T) { meta.ExitMessage = "previous exit" require.NoError(t, m.saveMetadata(meta)) - device := devices.VGPUDevice{ - Framework: devices.VGPUFrameworkVendorVFIO, - VFAddress: "0000:82:00.4", - ProfileType: "1148", - ProfileName: "NVIDIA L40S-2Q", - SysfsPath: "/sys/bus/pci/devices/0000:82:00.4", - } + device := testVendorVFIODevice("NVIDIA L40S-2Q") cause := errors.New("create verification and rollback failed") m.createVGPU = func(context.Context, string, string) (*devices.VGPUDevice, error) { return nil, &devices.VGPUCreateCleanupPendingError{Device: device, Err: cause} @@ -225,13 +224,7 @@ func TestStartReportsUnretainedVGPUWhenRetentionSaveFails(t *testing.T) { m, id := newStartRollbackVGPUManager(t, func(context.Context, devices.VGPUAssignment) error { return nil }) - device := devices.VGPUDevice{ - Framework: devices.VGPUFrameworkVendorVFIO, - VFAddress: "0000:82:00.4", - ProfileType: "1148", - ProfileName: "NVIDIA L40S-2Q", - SysfsPath: "/sys/bus/pci/devices/0000:82:00.4", - } + device := testVendorVFIODevice("NVIDIA L40S-2Q") cause := errors.New("create verification and rollback failed") m.createVGPU = func(context.Context, string, string) (*devices.VGPUDevice, error) { instanceDir := filepath.Dir(m.paths.InstanceMetadata(id))