feat(metrics): expose orphaned vGPU mdev gauge

lib/instances/metrics.go

@@ -5,6 +5,7 @@ import (
 	"strconv"
 	"time"
 
+	"github.com/kernel/hypeman/lib/devices"
 	"github.com/kernel/hypeman/lib/hypervisor"
 	mw "github.com/kernel/hypeman/lib/middleware"
 	hypotel "github.com/kernel/hypeman/lib/otel"
@@ -288,6 +289,14 @@ func newInstanceMetrics(meter metric.Meter, tracer trace.Tracer, m *manager) (*M
 		return nil, err
 	}
 
+	gpuMdevsTotal, err := meter.Int64ObservableGauge(
+		"hypeman_gpu_mdevs_total",
+		metric.WithDescription("Host vGPU mdev devices by whether an instance references them"),
+	)
+	if err != nil {
+		return nil, err
+	}
+
 	snapshotCompressionActiveTotal, err := meter.Int64ObservableGauge(
 		"hypeman_snapshot_compression_active_total",
 		metric.WithDescription("Total number of actively running snapshot compression jobs"),
@@ -359,10 +368,19 @@ func newInstanceMetrics(meter metric.Meter, tracer trace.Tracer, m *manager) (*M
 				o.ObserveInt64(instancesTotal, count, metric.WithAttributes(attrs...))
 				o.ObserveFloat64(oldestInStateSeconds, oldestAgeSeconds[key], metric.WithAttributes(attrs...))
 			}
+
+			mdevs, err := devices.ListMdevDevices()
+			if err != nil || len(mdevs) == 0 {
+				return nil
+			}
+			claimed, orphaned := countGPUMdevs(mdevs, instances)
+			o.ObserveInt64(gpuMdevsTotal, claimed, metric.WithAttributes(attribute.String("state", "claimed")))
+			o.ObserveInt64(gpuMdevsTotal, orphaned, metric.WithAttributes(attribute.String("state", "orphaned")))
 			return nil
 		},
 		instancesTotal,
 		oldestInStateSeconds,
+		gpuMdevsTotal,
 	)
 	if err != nil {
 		return nil, err
@@ -683,3 +701,23 @@ func (m *manager) recordLifecycleEventDropped(ctx context.Context, consumer Life
 		attribute.String("reason", string(reason)),
 	))
 }
+
+// countGPUMdevs splits host mdevs into those referenced by an instance and
+// those no instance references. An unreferenced mdev holds a vGPU slot that
+// nothing will release until the next startup reconciliation.
+func countGPUMdevs(mdevs []devices.MdevDevice, instances []Instance) (claimed, orphaned int64) {
+	referenced := make(map[string]struct{}, len(instances))
+	for _, inst := range instances {
+		if inst.GPUMdevUUID != "" {
+			referenced[inst.GPUMdevUUID] = struct{}{}
+		}
+	}
+	for _, mdev := range mdevs {
+		if _, ok := referenced[mdev.UUID]; ok {
+			claimed++
+		} else {
+			orphaned++
+		}
+	}
+	return claimed, orphaned
+}

lib/instances/metrics_gpu_test.go

@@ -0,0 +1,37 @@
+package instances
+
+import (
+	"testing"
+
+	"github.com/kernel/hypeman/lib/devices"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestCountGPUMdevs(t *testing.T) {
+	t.Parallel()
+
+	mdevs := []devices.MdevDevice{
+		{UUID: "claimed-running"},
+		{UUID: "claimed-stopped"},
+		{UUID: "orphaned-a"},
+		{UUID: "orphaned-b"},
+	}
+	instances := []Instance{
+		{StoredMetadata: StoredMetadata{GPUMdevUUID: "claimed-running"}, State: StateRunning},
+		{StoredMetadata: StoredMetadata{GPUMdevUUID: "claimed-stopped"}, State: StateStopped},
+		{StoredMetadata: StoredMetadata{GPUMdevUUID: "no-longer-on-host"}, State: StateStopped},
+		{State: StateRunning},
+	}
+
+	claimed, orphaned := countGPUMdevs(mdevs, instances)
+	assert.Equal(t, int64(2), claimed)
+	assert.Equal(t, int64(2), orphaned)
+
+	claimed, orphaned = countGPUMdevs(nil, instances)
+	assert.Equal(t, int64(0), claimed)
+	assert.Equal(t, int64(0), orphaned)
+
+	claimed, orphaned = countGPUMdevs(mdevs, nil)
+	assert.Equal(t, int64(0), claimed)
+	assert.Equal(t, int64(4), orphaned)
+}

lib/otel/README.md

@@ -100,6 +100,7 @@ This keeps pull and push views aligned because both are sourced from the same OT
 | `hypeman_resources_image_storage_bytes` | gauge | kind | Current and maximum image storage bytes |
 | `hypeman_resources_gpu_slots` | gauge | kind | Total and used GPU slots |
 | `hypeman_resources_gpu_profile_slots` | gauge | profile, kind | Available GPU slots by profile |
+| `hypeman_gpu_mdevs_total` | gauge | state | Host vGPU mdevs that an instance references (`claimed`) or that nothing references (`orphaned`) |
 
 ### Volumes
 | Metric | Type | Description |