diff --git a/lib/instances/metrics.go b/lib/instances/metrics.go index 1ada5ac1e..b1d62d482 100644 --- a/lib/instances/metrics.go +++ b/lib/instances/metrics.go @@ -5,6 +5,7 @@ import ( "strconv" "time" + "github.com/kernel/hypeman/lib/devices" "github.com/kernel/hypeman/lib/hypervisor" mw "github.com/kernel/hypeman/lib/middleware" hypotel "github.com/kernel/hypeman/lib/otel" @@ -288,6 +289,14 @@ func newInstanceMetrics(meter metric.Meter, tracer trace.Tracer, m *manager) (*M return nil, err } + gpuMdevsTotal, err := meter.Int64ObservableGauge( + "hypeman_gpu_mdevs_total", + metric.WithDescription("Host vGPU mdev devices by whether an instance references them"), + ) + if err != nil { + return nil, err + } + snapshotCompressionActiveTotal, err := meter.Int64ObservableGauge( "hypeman_snapshot_compression_active_total", metric.WithDescription("Total number of actively running snapshot compression jobs"), @@ -359,10 +368,19 @@ func newInstanceMetrics(meter metric.Meter, tracer trace.Tracer, m *manager) (*M o.ObserveInt64(instancesTotal, count, metric.WithAttributes(attrs...)) o.ObserveFloat64(oldestInStateSeconds, oldestAgeSeconds[key], metric.WithAttributes(attrs...)) } + + mdevs, err := devices.ListMdevDevices() + if err != nil || len(mdevs) == 0 { + return nil + } + claimed, orphaned := countGPUMdevs(mdevs, instances) + o.ObserveInt64(gpuMdevsTotal, claimed, metric.WithAttributes(attribute.String("state", "claimed"))) + o.ObserveInt64(gpuMdevsTotal, orphaned, metric.WithAttributes(attribute.String("state", "orphaned"))) return nil }, instancesTotal, oldestInStateSeconds, + gpuMdevsTotal, ) if err != nil { return nil, err @@ -683,3 +701,23 @@ func (m *manager) recordLifecycleEventDropped(ctx context.Context, consumer Life attribute.String("reason", string(reason)), )) } + +// countGPUMdevs splits host mdevs into those referenced by an instance and +// those no instance references. An unreferenced mdev holds a vGPU slot that +// nothing will release until the next startup reconciliation. +func countGPUMdevs(mdevs []devices.MdevDevice, instances []Instance) (claimed, orphaned int64) { + referenced := make(map[string]struct{}, len(instances)) + for _, inst := range instances { + if inst.GPUMdevUUID != "" { + referenced[inst.GPUMdevUUID] = struct{}{} + } + } + for _, mdev := range mdevs { + if _, ok := referenced[mdev.UUID]; ok { + claimed++ + } else { + orphaned++ + } + } + return claimed, orphaned +} diff --git a/lib/instances/metrics_gpu_test.go b/lib/instances/metrics_gpu_test.go new file mode 100644 index 000000000..26845c995 --- /dev/null +++ b/lib/instances/metrics_gpu_test.go @@ -0,0 +1,37 @@ +package instances + +import ( + "testing" + + "github.com/kernel/hypeman/lib/devices" + "github.com/stretchr/testify/assert" +) + +func TestCountGPUMdevs(t *testing.T) { + t.Parallel() + + mdevs := []devices.MdevDevice{ + {UUID: "claimed-running"}, + {UUID: "claimed-stopped"}, + {UUID: "orphaned-a"}, + {UUID: "orphaned-b"}, + } + instances := []Instance{ + {StoredMetadata: StoredMetadata{GPUMdevUUID: "claimed-running"}, State: StateRunning}, + {StoredMetadata: StoredMetadata{GPUMdevUUID: "claimed-stopped"}, State: StateStopped}, + {StoredMetadata: StoredMetadata{GPUMdevUUID: "no-longer-on-host"}, State: StateStopped}, + {State: StateRunning}, + } + + claimed, orphaned := countGPUMdevs(mdevs, instances) + assert.Equal(t, int64(2), claimed) + assert.Equal(t, int64(2), orphaned) + + claimed, orphaned = countGPUMdevs(nil, instances) + assert.Equal(t, int64(0), claimed) + assert.Equal(t, int64(0), orphaned) + + claimed, orphaned = countGPUMdevs(mdevs, nil) + assert.Equal(t, int64(0), claimed) + assert.Equal(t, int64(4), orphaned) +} diff --git a/lib/otel/README.md b/lib/otel/README.md index ed4613b68..bc05fcfcb 100644 --- a/lib/otel/README.md +++ b/lib/otel/README.md @@ -100,6 +100,7 @@ This keeps pull and push views aligned because both are sourced from the same OT | `hypeman_resources_image_storage_bytes` | gauge | kind | Current and maximum image storage bytes | | `hypeman_resources_gpu_slots` | gauge | kind | Total and used GPU slots | | `hypeman_resources_gpu_profile_slots` | gauge | profile, kind | Available GPU slots by profile | +| `hypeman_gpu_mdevs_total` | gauge | state | Host vGPU mdevs that an instance references (`claimed`) or that nothing references (`orphaned`) | ### Volumes | Metric | Type | Description |