ITQC/containernetworking-plugins
24
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 34 Wiki Activity

Merge pull request #642 from Nordix/dpdk-ipam

Browse Source 
host-device: add ipam support for dpdk device
This commit is contained in:
Dan Williams 2021-12-15 10:50:31 -06:00 committed by GitHub
commit b76849596f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 287 additions and 69 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

140
plugins/main/host-device/host-device.go
View File

@ -39,7 +39,7 @@ import (
bv "github.com/containernetworking/plugins/pkg/utils/buildversion"
)
const (
var (
sysBusPCI = "/sys/bus/pci/devices"
)
@ -49,8 +49,9 @@ var userspaceDrivers = []string{"vfio-pci", "uio_pci_generic", "igb_uio"}
//NetConf for host-device config, look the README to learn how to use those parameters
type NetConf struct {
types.NetConf
Device string `json:"device"` // Device-Name, something like eth0 or can0 etc.
HWAddr string `json:"hwaddr"` // MAC Address of target network interface
Device string `json:"device"` // Device-Name, something like eth0 or can0 etc.
HWAddr string `json:"hwaddr"` // MAC Address of target network interface
DPDKMode bool
KernelPath string `json:"kernelpath"` // Kernelpath of the device
PCIAddr string `json:"pciBusID"` // PCI Address of target network device
RuntimeConfig struct {
@ -67,7 +68,8 @@ func init() {
func loadConf(bytes []byte) (*NetConf, error) {
n := &NetConf{}
if err := json.Unmarshal(bytes, n); err != nil {
var err error
if err = json.Unmarshal(bytes, n); err != nil {
return nil, fmt.Errorf("failed to load netconf: %v", err)
}
@ -80,6 +82,13 @@ func loadConf(bytes []byte) (*NetConf, error) {
return nil, fmt.Errorf(`specify either "device", "hwaddr", "kernelpath" or "pciBusID"`)
}
if len(n.PCIAddr) > 0 {
n.DPDKMode, err = hasDpdkDriver(n.PCIAddr)
if err != nil {
return nil, fmt.Errorf("error with host device: %v", err)
}
}
return n, nil
}
@ -94,49 +103,17 @@ func cmdAdd(args *skel.CmdArgs) error {
}
defer containerNs.Close()
if len(cfg.PCIAddr) > 0 {
isDpdkMode, err := hasDpdkDriver(cfg.PCIAddr)
result := &current.Result{}
var contDev netlink.Link
if !cfg.DPDKMode {
hostDev, err := getLink(cfg.Device, cfg.HWAddr, cfg.KernelPath, cfg.PCIAddr)
if err != nil {
return fmt.Errorf("error with host device: %v", err)
return fmt.Errorf("failed to find host device: %v", err)
}
if isDpdkMode {
return types.PrintResult(&current.Result{}, cfg.CNIVersion)
}
}
hostDev, err := getLink(cfg.Device, cfg.HWAddr, cfg.KernelPath, cfg.PCIAddr)
if err != nil {
return fmt.Errorf("failed to find host device: %v", err)
}
contDev, err := moveLinkIn(hostDev, containerNs, args.IfName)
if err != nil {
return fmt.Errorf("failed to move link %v", err)
}
var result *current.Result
// run the IPAM plugin and get back the config to apply
if cfg.IPAM.Type != "" {
r, err := ipam.ExecAdd(cfg.IPAM.Type, args.StdinData)
contDev, err = moveLinkIn(hostDev, containerNs, args.IfName)
if err != nil {
return err
}
// Invoke ipam del if err to avoid ip leak
defer func() {
if err != nil {
ipam.ExecDel(cfg.IPAM.Type, args.StdinData)
}
}()
// Convert whatever the IPAM result was into the current Result type
result, err = current.NewResultFromResult(r)
if err != nil {
return err
}
if len(result.IPs) == 0 {
return errors.New("IPAM plugin returned missing IP config")
return fmt.Errorf("failed to move link %v", err)
}
result.Interfaces = []*current.Interface{{
@ -144,13 +121,48 @@ func cmdAdd(args *skel.CmdArgs) error {
Mac: contDev.Attrs().HardwareAddr.String(),
Sandbox: containerNs.Path(),
}}
for _, ipc := range result.IPs {
// All addresses apply to the container interface (move from host)
ipc.Interface = current.Int(0)
}
}
if cfg.IPAM.Type == "" {
if cfg.DPDKMode {
return types.PrintResult(result, cfg.CNIVersion)
}
return printLink(contDev, cfg.CNIVersion, containerNs)
}
// run the IPAM plugin and get back the config to apply
r, err := ipam.ExecAdd(cfg.IPAM.Type, args.StdinData)
if err != nil {
return err
}
// Invoke ipam del if err to avoid ip leak
defer func() {
if err != nil {
ipam.ExecDel(cfg.IPAM.Type, args.StdinData)
}
}()
// Convert whatever the IPAM result was into the current Result type
newResult, err := current.NewResultFromResult(r)
if err != nil {
return err
}
if len(newResult.IPs) == 0 {
return errors.New("IPAM plugin returned missing IP config")
}
for _, ipc := range newResult.IPs {
// All addresses apply to the container interface (move from host)
ipc.Interface = current.Int(0)
}
newResult.Interfaces = result.Interfaces
if !cfg.DPDKMode {
err = containerNs.Do(func(_ ns.NetNS) error {
if err := ipam.ConfigureIface(args.IfName, result); err != nil {
if err := ipam.ConfigureIface(args.IfName, newResult); err != nil {
return err
}
return nil
@ -158,13 +170,11 @@ func cmdAdd(args *skel.CmdArgs) error {
if err != nil {
return err
}
result.DNS = cfg.DNS
return types.PrintResult(result, cfg.CNIVersion)
}
return printLink(contDev, cfg.CNIVersion, containerNs)
newResult.DNS = cfg.DNS
return types.PrintResult(newResult, cfg.CNIVersion)
}
func cmdDel(args *skel.CmdArgs) error {
@ -181,26 +191,18 @@ func cmdDel(args *skel.CmdArgs) error {
}
defer containerNs.Close()
if len(cfg.PCIAddr) > 0 {
isDpdkMode, err := hasDpdkDriver(cfg.PCIAddr)
if err != nil {
return fmt.Errorf("error with host device: %v", err)
}
if isDpdkMode {
return nil
}
}
if err := moveLinkOut(containerNs, args.IfName); err != nil {
return err
}
if cfg.IPAM.Type != "" {
if err := ipam.ExecDel(cfg.IPAM.Type, args.StdinData); err != nil {
return err
}
}
if !cfg.DPDKMode {
if err := moveLinkOut(containerNs, args.IfName); err != nil {
return err
}
}
return nil
}
@ -410,6 +412,10 @@ func cmdCheck(args *skel.CmdArgs) error {
return err
}
if cfg.DPDKMode {
return nil
}
var contMap current.Interface
// Find interfaces for name we know, that of host-device inside container
for _, intf := range result.Interfaces {

216
plugins/main/host-device/host-device_test.go
View File

@ -17,14 +17,17 @@ package main
import (
"encoding/json"
"fmt"
"io/ioutil"
"math/rand"
"net"
"os"
"path"
"strings"
"github.com/containernetworking/cni/pkg/skel"
"github.com/containernetworking/cni/pkg/types"
"github.com/containernetworking/cni/pkg/types/040"
"github.com/containernetworking/cni/pkg/types/100"
types040 "github.com/containernetworking/cni/pkg/types/040"
types100 "github.com/containernetworking/cni/pkg/types/100"
"github.com/containernetworking/cni/pkg/version"
"github.com/containernetworking/plugins/pkg/ns"
"github.com/containernetworking/plugins/pkg/testutils"
@ -217,6 +220,7 @@ func buildOneConfig(name, cniVersion string, orig *Net, prevResult types.Result)
type tester interface {
expectInterfaces(result types.Result, name, mac, sandbox string)
expectDpdkInterfaceIP(result types.Result, ipAddress string)
}
type testerBase struct{}
@ -252,6 +256,15 @@ func (t *testerV10x) expectInterfaces(result types.Result, name, mac, sandbox st
}))
}
func (t *testerV10x) expectDpdkInterfaceIP(result types.Result, ipAddress string) {
// check that the result was sane
res, err := types100.NewResultFromResult(result)
Expect(err).NotTo(HaveOccurred())
Expect(len(res.Interfaces)).To(Equal(0))
Expect(len(res.IPs)).To(Equal(1))
Expect(res.IPs[0].Address.String()).To(Equal(ipAddress))
}
func (t *testerV04x) expectInterfaces(result types.Result, name, mac, sandbox string) {
// check that the result was sane
res, err := types040.NewResultFromResult(result)
@ -265,6 +278,15 @@ func (t *testerV04x) expectInterfaces(result types.Result, name, mac, sandbox st
}))
}
func (t *testerV04x) expectDpdkInterfaceIP(result types.Result, ipAddress string) {
// check that the result was sane
res, err := types040.NewResultFromResult(result)
Expect(err).NotTo(HaveOccurred())
Expect(len(res.Interfaces)).To(Equal(0))
Expect(len(res.IPs)).To(Equal(1))
Expect(res.IPs[0].Address.String()).To(Equal(ipAddress))
}
func (t *testerV03x) expectInterfaces(result types.Result, name, mac, sandbox string) {
// check that the result was sane
res, err := types040.NewResultFromResult(result)
@ -278,6 +300,15 @@ func (t *testerV03x) expectInterfaces(result types.Result, name, mac, sandbox st
}))
}
func (t *testerV03x) expectDpdkInterfaceIP(result types.Result, ipAddress string) {
// check that the result was sane
res, err := types040.NewResultFromResult(result)
Expect(err).NotTo(HaveOccurred())
Expect(len(res.Interfaces)).To(Equal(0))
Expect(len(res.IPs)).To(Equal(1))
Expect(res.IPs[0].Address.String()).To(Equal(ipAddress))
}
var _ = Describe("base functionality", func() {
var originalNS, targetNS ns.NetNS
var ifname string
@ -509,6 +540,65 @@ var _ = Describe("base functionality", func() {
})
})
It(fmt.Sprintf("Works with a valid %s config on a DPDK device with IPAM", ver), func() {
fs := &fakeFilesystem{
dirs: []string{
"sys/bus/pci/devices/0000:00:00.1",
"sys/bus/pci/drivers/vfio-pci",
},
symlinks: map[string]string{
"sys/bus/pci/devices/0000:00:00.1/driver": "../../../../bus/pci/drivers/vfio-pci",
},
}
defer fs.use()()
// call CmdAdd
targetIP := "10.10.0.1/24"
cniName := "eth0"
conf := fmt.Sprintf(`{
"cniVersion": "%s",
"name": "cni-plugin-host-device-test",
"type": "host-device",
"ipam": {
"type": "static",
"addresses": [
{
"address":"`+targetIP+`",
"gateway": "10.10.0.254"
}]
},
"pciBusID": %q
}`, ver, "0000:00:00.1")
args := &skel.CmdArgs{
ContainerID: "dummy",
IfName: cniName,
Netns: targetNS.Path(),
StdinData: []byte(conf),
}
var resI types.Result
err := originalNS.Do(func(ns.NetNS) error {
defer GinkgoRecover()
var err error
resI, _, err = testutils.CmdAddWithArgs(args, func() error { return cmdAdd(args) })
return err
})
Expect(err).NotTo(HaveOccurred())
// check that the result was sane
t := newTesterByVersion(ver)
t.expectDpdkInterfaceIP(resI, targetIP)
// call CmdDel
_ = originalNS.Do(func(ns.NetNS) error {
defer GinkgoRecover()
err = testutils.CmdDelWithArgs(args, func() error {
return cmdDel(args)
})
Expect(err).NotTo(HaveOccurred())
return nil
})
})
It(fmt.Sprintf("Works with a valid %s config with IPAM", ver), func() {
var origLink netlink.Link
@ -721,6 +811,89 @@ var _ = Describe("base functionality", func() {
})
})
It(fmt.Sprintf("Works with a valid %s config on a DPDK device with IPAM", ver), func() {
fs := &fakeFilesystem{
dirs: []string{
"sys/bus/pci/devices/0000:00:00.1",
"sys/bus/pci/drivers/vfio-pci",
},
symlinks: map[string]string{
"sys/bus/pci/devices/0000:00:00.1/driver": "../../../../bus/pci/drivers/vfio-pci",
},
}
defer fs.use()()
// call CmdAdd
targetIP := "10.10.0.1/24"
cniName := "eth0"
conf := fmt.Sprintf(`{
"cniVersion": "%s",
"name": "cni-plugin-host-device-test",
"type": "host-device",
"ipam": {
"type": "static",
"addresses": [
{
"address":"`+targetIP+`",
"gateway": "10.10.0.254"
}]
},
"pciBusID": %q
}`, ver, "0000:00:00.1")
args := &skel.CmdArgs{
ContainerID: "dummy",
Netns: targetNS.Path(),
IfName: cniName,
StdinData: []byte(conf),
}
var resI types.Result
err := originalNS.Do(func(ns.NetNS) error {
defer GinkgoRecover()
var err error
resI, _, err = testutils.CmdAddWithArgs(args, func() error { return cmdAdd(args) })
return err
})
Expect(err).NotTo(HaveOccurred())
// check that the result was sane
t := newTesterByVersion(ver)
t.expectDpdkInterfaceIP(resI, targetIP)
// call CmdCheck
n := &Net{}
err = json.Unmarshal([]byte(conf), &n)
Expect(err).NotTo(HaveOccurred())
n.IPAM, _, err = LoadIPAMConfig([]byte(conf), "")
Expect(err).NotTo(HaveOccurred())
if testutils.SpecVersionHasCHECK(ver) {
newConf, err := buildOneConfig("testConfig", ver, n, resI)
Expect(err).NotTo(HaveOccurred())
confString, err := json.Marshal(newConf)
Expect(err).NotTo(HaveOccurred())
args.StdinData = confString
err = originalNS.Do(func(ns.NetNS) error {
defer GinkgoRecover()
return testutils.CmdCheckWithArgs(args, func() error { return cmdCheck(args) })
})
Expect(err).NotTo(HaveOccurred())
}
// call CmdDel
_ = originalNS.Do(func(ns.NetNS) error {
defer GinkgoRecover()
err = testutils.CmdDelWithArgs(args, func() error {
return cmdDel(args)
})
Expect(err).NotTo(HaveOccurred())
return nil
})
})
It(fmt.Sprintf("Works with a valid %s config with IPAM", ver), func() {
var origLink netlink.Link
@ -971,3 +1144,42 @@ var _ = Describe("base functionality", func() {
})
}
})
type fakeFilesystem struct {
rootDir string
dirs []string
symlinks map[string]string
}
func (fs *fakeFilesystem) use() func() {
// create the new fake fs root dir in /tmp/sriov...
tmpDir, err := ioutil.TempDir("", "sriov")
if err != nil {
panic(fmt.Errorf("error creating fake root dir: %s", err.Error()))
}
fs.rootDir = tmpDir
for _, dir := range fs.dirs {
err := os.MkdirAll(path.Join(fs.rootDir, dir), 0755)
if err != nil {
panic(fmt.Errorf("error creating fake directory: %s", err.Error()))
}
}
for link, target := range fs.symlinks {
err = os.Symlink(target, path.Join(fs.rootDir, link))
if err != nil {
panic(fmt.Errorf("error creating fake symlink: %s", err.Error()))
}
}
sysBusPCI = path.Join(fs.rootDir, "/sys/bus/pci/devices")
return func() {
// remove temporary fake fs
err := os.RemoveAll(fs.rootDir)
if err != nil {
panic(fmt.Errorf("error tearing down fake filesystem: %s", err.Error()))
}
}
}