ITQC/containernetworking-plugins
24
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 34 Wiki Activity

Support DeviceID on Auxiliary Bus

Browse Source 
Device plugins may allocate network device on a bus
different than PCI.

sriov-network-device-plugin supports the allocation
of network devices over Auxiliary bus[1][2][3].

extend host-device CNI to support such devices if provided
through runtime config.

- Check if device provided by DeviceID runtime config
  is present on either PCI bus or Auxiliary bus
- extend getLink method to support getting netdev link obj
  from auxiliary bus
- add unit-test to cover the new flow

[1] https://github.com/k8snetworkplumbingwg/sriov-network-device-plugin/tree/master?tab=readme-ov-file#auxiliary-network-devices-selectors
[2] https://github.com/k8snetworkplumbingwg/sriov-network-device-plugin/tree/master/docs/subfunctions
[3] https://docs.kernel.org/networking/devlink/devlink-port.html

Signed-off-by: adrianc <adrianc@nvidia.com>
This commit is contained in:
adrianc 2024-01-24 17:19:17 +02:00
parent 8fc26ce7a0
commit d34720b531
No known key found for this signature in database
GPG Key ID: CC911D9D88FC463E
2 changed files with 124 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

89
plugins/main/host-device/host-device.go
View File

@ -37,7 +37,10 @@ import (
bv "github.com/containernetworking/plugins/pkg/utils/buildversion"
)
var sysBusPCI = "/sys/bus/pci/devices"
var (
sysBusPCI = "/sys/bus/pci/devices"
sysBusAuxiliary = "/sys/bus/auxiliary/devices"
)
// Array of different linux drivers bound to network device needed for DPDK
var userspaceDrivers = []string{"vfio-pci", "uio_pci_generic", "igb_uio"}
@ -53,6 +56,9 @@ type NetConf struct {
RuntimeConfig struct {
DeviceID string `json:"deviceID,omitempty"`
} `json:"runtimeConfig,omitempty"`
// for internal use
auxDevice string `json:"-"` // Auxiliary device name as appears on Auxiliary bus (/sys/bus/auxiliary)
}
func init() {
@ -62,6 +68,31 @@ func init() {
runtime.LockOSThread()
}
// handleDeviceID updates netconf fields with DeviceID runtime config
func handleDeviceID(netconf *NetConf) error {
deviceID := netconf.RuntimeConfig.DeviceID
if deviceID == "" {
return nil
}
// Check if deviceID is a PCI device
pciPath := filepath.Join(sysBusPCI, deviceID)
if _, err := os.Stat(pciPath); err == nil {
netconf.PCIAddr = deviceID
return nil
}
// Check if deviceID is an Auxiliary device
auxPath := filepath.Join(sysBusAuxiliary, deviceID)
if _, err := os.Stat(auxPath); err == nil {
netconf.PCIAddr = ""
netconf.auxDevice = deviceID
return nil
}
return fmt.Errorf("runtime config DeviceID %s not found or unsupported", deviceID)
}
func loadConf(bytes []byte) (*NetConf, error) {
n := &NetConf{}
var err error
@ -69,12 +100,12 @@ func loadConf(bytes []byte) (*NetConf, error) {
return nil, fmt.Errorf("failed to load netconf: %v", err)
}
if n.RuntimeConfig.DeviceID != "" {
// Override PCI device with the standardized DeviceID provided in Runtime Config.
n.PCIAddr = n.RuntimeConfig.DeviceID
// Override device with the standardized DeviceID if provided in Runtime Config.
if err := handleDeviceID(n); err != nil {
return nil, err
}
if n.Device == "" && n.HWAddr == "" && n.KernelPath == "" && n.PCIAddr == "" {
if n.Device == "" && n.HWAddr == "" && n.KernelPath == "" && n.PCIAddr == "" && n.auxDevice == "" {
return nil, fmt.Errorf(`specify either "device", "hwaddr", "kernelpath" or "pciBusID"`)
}
@ -102,7 +133,7 @@ func cmdAdd(args *skel.CmdArgs) error {
result := &current.Result{}
var contDev netlink.Link
if !cfg.DPDKMode {
hostDev, err := getLink(cfg.Device, cfg.HWAddr, cfg.KernelPath, cfg.PCIAddr)
hostDev, err := getLink(cfg.Device, cfg.HWAddr, cfg.KernelPath, cfg.PCIAddr, cfg.auxDevice)
if err != nil {
return fmt.Errorf("failed to find host device: %v", err)
}
@ -314,11 +345,19 @@ func printLink(dev netlink.Link, cniVersion string, containerNs ns.NetNS) error
return types.PrintResult(&result, cniVersion)
}
func getLink(devname, hwaddr, kernelpath, pciaddr string) (netlink.Link, error) {
links, err := netlink.LinkList()
func linkFromPath(path string) (netlink.Link, error) {
entries, err := os.ReadDir(path)
if err != nil {
return nil, fmt.Errorf("failed to list node links: %v", err)
return nil, fmt.Errorf("failed to read directory %s: %q", path, err)
}
if len(entries) > 0 {
// grab the first net device
return netlink.LinkByName(entries[0].Name())
}
return nil, fmt.Errorf("failed to find network device in path %s", path)
}
func getLink(devname, hwaddr, kernelpath, pciaddr string, auxDev string) (netlink.Link, error) {
switch {
case len(devname) > 0:
@ -329,6 +368,11 @@ func getLink(devname, hwaddr, kernelpath, pciaddr string) (netlink.Link, error)
return nil, fmt.Errorf("failed to parse MAC address %q: %v", hwaddr, err)
}
links, err := netlink.LinkList()
if err != nil {
return nil, fmt.Errorf("failed to list node links: %v", err)
}
for _, link := range links {
if bytes.Equal(link.Attrs().HardwareAddr, hwAddr) {
return link, nil
@ -339,20 +383,7 @@ func getLink(devname, hwaddr, kernelpath, pciaddr string) (netlink.Link, error)
return nil, fmt.Errorf("kernel device path %q must be absolute and begin with /sys/devices/", kernelpath)
}
netDir := filepath.Join(kernelpath, "net")
entries, err := os.ReadDir(netDir)
if err != nil {
return nil, fmt.Errorf("failed to find network devices at %q", netDir)
}
// Grab the first device from eg /sys/devices/pci0000:00/0000:00:19.0/net
for _, entry := range entries {
// Make sure it's really an interface
for _, l := range links {
if entry.Name() == l.Attrs().Name {
return l, nil
}
}
}
return linkFromPath(netDir)
case len(pciaddr) > 0:
netDir := filepath.Join(sysBusPCI, pciaddr, "net")
if _, err := os.Lstat(netDir); err != nil {
@ -363,14 +394,10 @@ func getLink(devname, hwaddr, kernelpath, pciaddr string) (netlink.Link, error)
}
netDir = matches[0]
}
entries, err := os.ReadDir(netDir)
if err != nil {
return nil, fmt.Errorf("failed to read net directory %s: %q", netDir, err)
}
if len(entries) > 0 {
return netlink.LinkByName(entries[0].Name())
}
return nil, fmt.Errorf("failed to find device name for pci address %s", pciaddr)
return linkFromPath(netDir)
case len(auxDev) > 0:
netDir := filepath.Join(sysBusAuxiliary, auxDev, "net")
return linkFromPath(netDir)
}
return nil, fmt.Errorf("failed to find physical interface")

66
plugins/main/host-device/host-device_test.go
View File

@ -898,6 +898,71 @@ var _ = Describe("base functionality", func() {
})
})
It(fmt.Sprintf("Works with a valid %s config on auxiliary device", ver), func() {
var origLink netlink.Link
ifname := "eth0"
fs := &fakeFilesystem{
dirs: []string{
fmt.Sprintf("sys/bus/auxiliary/devices/mlx5_core.sf.4/net/%s", ifname),
},
}
defer fs.use()()
// prepare ifname in original namespace
_ = originalNS.Do(func(ns.NetNS) error {
defer GinkgoRecover()
err := netlink.LinkAdd(&netlink.Dummy{
LinkAttrs: netlink.LinkAttrs{
Name: ifname,
},
})
Expect(err).NotTo(HaveOccurred())
origLink, err = netlink.LinkByName(ifname)
Expect(err).NotTo(HaveOccurred())
err = netlink.LinkSetUp(origLink)
Expect(err).NotTo(HaveOccurred())
return nil
})
// call CmdAdd
cniName := "net1"
conf := fmt.Sprintf(`{
"cniVersion": "%s",
"name": "cni-plugin-host-device-test",
"type": "host-device",
"runtimeConfig": {"deviceID": %q}
}`, ver, "mlx5_core.sf.4")
args := &skel.CmdArgs{
ContainerID: "dummy",
IfName: cniName,
Netns: targetNS.Path(),
StdinData: []byte(conf),
}
var resI types.Result
err := originalNS.Do(func(ns.NetNS) error {
defer GinkgoRecover()
var err error
resI, _, err = testutils.CmdAddWithArgs(args, func() error { return cmdAdd(args) })
return err
})
Expect(err).NotTo(HaveOccurred())
// check that the result was sane
t := newTesterByVersion(ver)
t.expectInterfaces(resI, cniName, origLink.Attrs().HardwareAddr.String(), targetNS.Path())
// call CmdDel
_ = originalNS.Do(func(ns.NetNS) error {
defer GinkgoRecover()
err = testutils.CmdDelWithArgs(args, func() error {
return cmdDel(args)
})
Expect(err).NotTo(HaveOccurred())
return nil
})
})
It(fmt.Sprintf("Works with a valid %s config with IPAM", ver), func() {
var origLink netlink.Link
@ -1181,6 +1246,7 @@ func (fs *fakeFilesystem) use() func() {
}
sysBusPCI = path.Join(fs.rootDir, "/sys/bus/pci/devices")
sysBusAuxiliary = path.Join(fs.rootDir, "/sys/bus/auxiliary/devices")
return func() {
// remove temporary fake fs