Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
52 changes: 52 additions & 0 deletions pkg/device/attributes.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
/*
Copyright 2026 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package device

import (
"github.com/kubernetes-sigs/dra-driver-cpu/pkg/cpuinfo"
resourceapi "k8s.io/api/resource/v1"
"k8s.io/utils/cpuset"
)

func MakeIndividualAttributes(cpu cpuinfo.CPUInfo) map[resourceapi.QualifiedName]resourceapi.DeviceAttribute {
numaNode := int64(cpu.NUMANodeID)
cacheL3ID := int64(cpu.UncoreCacheID)
socketID := int64(cpu.SocketID)
coreID := int64(cpu.CoreID)
cpuID := int64(cpu.CpuID)
coreType := cpu.CoreType.String()
return map[resourceapi.QualifiedName]resourceapi.DeviceAttribute{
"dra.cpu/numaNodeID": {IntValue: &numaNode},
"dra.cpu/cacheL3ID": {IntValue: &cacheL3ID},
"dra.cpu/coreType": {StringValue: &coreType},
"dra.cpu/socketID": {IntValue: &socketID},
"dra.cpu/coreID": {IntValue: &coreID},
"dra.cpu/cpuID": {IntValue: &cpuID},
// TODO(pravk03): Remove. Hack to align with NIC (DRANet). We need some standard attribute to align other resources with CPU.
"dra.net/numaNode": {IntValue: &numaNode},
}
}

func MakeGroupedAttributes(topo *cpuinfo.CPUTopology, socketID int64, allocatableCPUs cpuset.CPUSet) map[resourceapi.QualifiedName]resourceapi.DeviceAttribute {
smtEnabled := topo.SMTEnabled
availableCPUs := int64(allocatableCPUs.Size())
return map[resourceapi.QualifiedName]resourceapi.DeviceAttribute{
"dra.cpu/socketID": {IntValue: &socketID},
"dra.cpu/numCPUs": {IntValue: &availableCPUs},
"dra.cpu/smtEnabled": {BoolValue: &smtEnabled},
}
}
32 changes: 32 additions & 0 deletions pkg/device/consts.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
/*
Copyright 2026 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package device

const (
// maxDevicesPerResourceSlice is the maximum number of devices that can be packed into a single
// ResourceSlice object. This is a hard limit defined in the Kubernetes API at
// https://github.com/kubernetes/kubernetes/blob/8e6d788887034b799f6c2a86991a68a080bb0576/pkg/apis/resource/types.go#L245
maxDevicesPerResourceSlice = 128
cpuDevicePrefix = "cpudev"

// Grouped Mode
// cpuResourceQualifiedName is the qualified name for the CPU resource capacity.
cpuResourceQualifiedName = "dra.cpu/cpu"

cpuDeviceSocketGroupedPrefix = "cpudevsocket"
cpuDeviceNUMAGroupedPrefix = "cpudevnuma"
)
133 changes: 133 additions & 0 deletions pkg/device/grouped_numa.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,133 @@
/*
Copyright 2026 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package device

import (
"fmt"

"github.com/kubernetes-sigs/dra-driver-cpu/pkg/cpuinfo"
"github.com/kubernetes-sigs/dra-driver-cpu/pkg/cpumanager"
resourceapi "k8s.io/api/resource/v1"
"k8s.io/apimachinery/pkg/api/resource"
"k8s.io/klog/v2"
"k8s.io/utils/cpuset"
"k8s.io/utils/ptr"
)

type NUMAGroupedManager struct {
driverName string
cpuTopology *cpuinfo.CPUTopology
reservedCPUs cpuset.CPUSet
getSharedCPUs func() cpuset.CPUSet
deviceNameToNUMANodeID map[string]int
}

func NewNUMAGroupedManager(name string, topo *cpuinfo.CPUTopology, resv cpuset.CPUSet, getSharedCPUs func() cpuset.CPUSet) *NUMAGroupedManager {
return &NUMAGroupedManager{
driverName: name,
cpuTopology: topo,
reservedCPUs: resv,
getSharedCPUs: getSharedCPUs,
deviceNameToNUMANodeID: make(map[string]int),
}
}

func (mgr *NUMAGroupedManager) CreateSlices(_ klog.Logger) [][]resourceapi.Device {
klog.Info("Creating grouped CPU devices", "groupBy", "NUMANode")
var devices []resourceapi.Device

numaNodeIDs := mgr.cpuTopology.CPUDetails.NUMANodes().List()
for _, numaIDInt := range numaNodeIDs {
numaID := int64(numaIDInt)
deviceName := fmt.Sprintf("%s%03d", cpuDeviceNUMAGroupedPrefix, numaIDInt)
numaNodeCPUSet := mgr.cpuTopology.CPUDetails.CPUsInNUMANodes(numaIDInt)
allocatableCPUs := numaNodeCPUSet.Difference(mgr.reservedCPUs)
availableCPUsInNUMANode := int64(allocatableCPUs.Size())

if allocatableCPUs.Size() == 0 {
continue
}

// All CPUs in a NUMA node belong to the same socket.
anyCPU := allocatableCPUs.UnsortedList()[0]
socketID := int64(mgr.cpuTopology.CPUDetails[anyCPU].SocketID)

deviceCapacity := map[resourceapi.QualifiedName]resourceapi.DeviceCapacity{
cpuResourceQualifiedName: {Value: *resource.NewQuantity(availableCPUsInNUMANode, resource.DecimalSI)},
}

mgr.deviceNameToNUMANodeID[deviceName] = numaIDInt

deviceAttributes := MakeGroupedAttributes(mgr.cpuTopology, socketID, allocatableCPUs)
deviceAttributes["dra.cpu/numaNodeID"] = resourceapi.DeviceAttribute{IntValue: &numaID}
// TODO(pravk03): Remove. Hack to align with NIC (DRANet). We need some standard attribute to align other resources with CPU.
deviceAttributes["dra.net/numaNode"] = resourceapi.DeviceAttribute{IntValue: &numaID}

devices = append(devices, resourceapi.Device{
Name: deviceName,
Attributes: deviceAttributes,
Capacity: deviceCapacity,
AllowMultipleAllocations: ptr.To(true),
})
}

if len(devices) == 0 {
return nil
}
return [][]resourceapi.Device{devices}
}

func (mgr *NUMAGroupedManager) AllocateCPUs(logger klog.Logger, claim *resourceapi.ResourceClaim) (cpuset.CPUSet, error) {
logger = klog.LoggerWithValues(logger, "claim", claim.Namespace+"/"+claim.Name)

var cpuAssignment cpuset.CPUSet

for _, alloc := range claim.Status.Allocation.Devices.Results {
claimCPUCount := int64(0)
if alloc.Driver != mgr.driverName {
continue
}
if quantity, ok := alloc.ConsumedCapacity[cpuResourceQualifiedName]; ok {
count := quantity.Value()
claimCPUCount = count
logger.Info("Found CPUs request", "CPUCount", count, "device", alloc.Device)
}

var availableCPUsForDevice cpuset.CPUSet
numaNodeID, ok := mgr.deviceNameToNUMANodeID[alloc.Device]
if !ok {
return cpuset.CPUSet{}, fmt.Errorf("no valid NUMA node ID found for device %s", alloc.Device)
}
numaCPUs := mgr.cpuTopology.CPUDetails.CPUsInNUMANodes(numaNodeID)
availableCPUsForDevice = mgr.getSharedCPUs().Intersection(numaCPUs)
logger.Info("available CPUs", "NUMANode", numaNodeID, "totalCPUs", numaCPUs.String(), "availableCPUs", availableCPUsForDevice.String())

cur, err := cpumanager.TakeByTopologyNUMAPacked(logger, mgr.cpuTopology, availableCPUsForDevice, int(claimCPUCount), cpumanager.CPUSortingStrategyPacked, true)
if err != nil {
return cpuset.CPUSet{}, err
}
cpuAssignment = cpuAssignment.Union(cur)
logger.Info("CPU assignment", "device", alloc.Device, "partialCPUs", cur.String(), "totalCPUs", cpuAssignment.String())
}

if cpuAssignment.Size() == 0 {
logger.V(5).Info("AllocateCPUs no CPU allocations for this driver")
return cpuset.CPUSet{}, nil
}

return cpuAssignment, nil
}
124 changes: 124 additions & 0 deletions pkg/device/grouped_socket.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,124 @@
/*
Copyright 2026 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package device

import (
"fmt"

"github.com/kubernetes-sigs/dra-driver-cpu/pkg/cpuinfo"
"github.com/kubernetes-sigs/dra-driver-cpu/pkg/cpumanager"
resourceapi "k8s.io/api/resource/v1"
"k8s.io/apimachinery/pkg/api/resource"
"k8s.io/klog/v2"
"k8s.io/utils/cpuset"
"k8s.io/utils/ptr"
)

type SocketGroupedManager struct {
driverName string
cpuTopology *cpuinfo.CPUTopology
reservedCPUs cpuset.CPUSet
getSharedCPUs func() cpuset.CPUSet
deviceNameToSocketID map[string]int
}

func NewSocketGroupedManager(name string, topo *cpuinfo.CPUTopology, resv cpuset.CPUSet, getSharedCPUs func() cpuset.CPUSet) *SocketGroupedManager {
return &SocketGroupedManager{
driverName: name,
cpuTopology: topo,
reservedCPUs: resv,
getSharedCPUs: getSharedCPUs,
deviceNameToSocketID: make(map[string]int),
}
}

func (mgr *SocketGroupedManager) CreateSlices(logger klog.Logger) [][]resourceapi.Device {
logger.Info("Creating grouped CPU devices", "groupBy", "Socket")
var devices []resourceapi.Device

socketIDs := mgr.cpuTopology.CPUDetails.Sockets().List()
for _, socketIDInt := range socketIDs {
socketID := int64(socketIDInt)
deviceName := fmt.Sprintf("%s%03d", cpuDeviceSocketGroupedPrefix, socketIDInt)
socketCPUSet := mgr.cpuTopology.CPUDetails.CPUsInSockets(socketIDInt)
allocatableCPUs := socketCPUSet.Difference(mgr.reservedCPUs)
availableCPUsInSocket := int64(allocatableCPUs.Size())

if allocatableCPUs.Size() == 0 {
continue
}

deviceCapacity := map[resourceapi.QualifiedName]resourceapi.DeviceCapacity{
cpuResourceQualifiedName: {Value: *resource.NewQuantity(availableCPUsInSocket, resource.DecimalSI)},
}

mgr.deviceNameToSocketID[deviceName] = socketIDInt

devices = append(devices, resourceapi.Device{
Name: deviceName,
Attributes: MakeGroupedAttributes(mgr.cpuTopology, socketID, allocatableCPUs),
Capacity: deviceCapacity,
AllowMultipleAllocations: ptr.To(true),
})
}

if len(devices) == 0 {
return nil
}
return [][]resourceapi.Device{devices}
}

func (mgr *SocketGroupedManager) AllocateCPUs(logger klog.Logger, claim *resourceapi.ResourceClaim) (cpuset.CPUSet, error) {
logger = klog.LoggerWithValues(logger, "claim", claim.Namespace+"/"+claim.Name)

var cpuAssignment cpuset.CPUSet

for _, alloc := range claim.Status.Allocation.Devices.Results {
claimCPUCount := int64(0)
if alloc.Driver != mgr.driverName {
continue
}
if quantity, ok := alloc.ConsumedCapacity[cpuResourceQualifiedName]; ok {
count := quantity.Value()
claimCPUCount = count
logger.Info("Found CPUs request", "CPUCount", count, "device", alloc.Device)
}

var availableCPUsForDevice cpuset.CPUSet
socketID, ok := mgr.deviceNameToSocketID[alloc.Device]
if !ok {
return cpuset.CPUSet{}, fmt.Errorf("no valid socket ID found for device %s", alloc.Device)
}
socketCPUs := mgr.cpuTopology.CPUDetails.CPUsInSockets(socketID)
availableCPUsForDevice = mgr.getSharedCPUs().Intersection(socketCPUs)
logger.Info("available CPUs", "Socket", socketID, "totalCPUs", socketCPUs.String(), "availableCPUs", availableCPUsForDevice.String())

cur, err := cpumanager.TakeByTopologyNUMAPacked(logger, mgr.cpuTopology, availableCPUsForDevice, int(claimCPUCount), cpumanager.CPUSortingStrategyPacked, true)
if err != nil {
return cpuset.CPUSet{}, err
}
cpuAssignment = cpuAssignment.Union(cur)
logger.Info("CPU assignment", "device", alloc.Device, "partialCPUs", cur.String(), "totalCPUs", cpuAssignment.String())
}

if cpuAssignment.Size() == 0 {
logger.V(5).Info("AllocateCPUs no CPU allocations for this driver")
return cpuset.CPUSet{}, nil
}

return cpuAssignment, nil
}
Loading
Loading