From 3033fd2e75d7b7c3ba5e524a8c869e7cac150719 Mon Sep 17 00:00:00 2001 From: Periyasamy Palanisamy Date: Tue, 13 Jul 2021 12:07:18 +0200 Subject: [PATCH 1/2] add ipam support for dpdk device Signed-off-by: Periyasamy Palanisamy --- plugins/main/host-device/host-device.go | 138 ++++++++++++------------ 1 file changed, 72 insertions(+), 66 deletions(-) diff --git a/plugins/main/host-device/host-device.go b/plugins/main/host-device/host-device.go index f2c74f99..f6fc35c7 100644 --- a/plugins/main/host-device/host-device.go +++ b/plugins/main/host-device/host-device.go @@ -49,8 +49,9 @@ var userspaceDrivers = []string{"vfio-pci", "uio_pci_generic", "igb_uio"} //NetConf for host-device config, look the README to learn how to use those parameters type NetConf struct { types.NetConf - Device string `json:"device"` // Device-Name, something like eth0 or can0 etc. - HWAddr string `json:"hwaddr"` // MAC Address of target network interface + Device string `json:"device"` // Device-Name, something like eth0 or can0 etc. + HWAddr string `json:"hwaddr"` // MAC Address of target network interface + DPDKMode bool KernelPath string `json:"kernelpath"` // Kernelpath of the device PCIAddr string `json:"pciBusID"` // PCI Address of target network device RuntimeConfig struct { @@ -67,7 +68,8 @@ func init() { func loadConf(bytes []byte) (*NetConf, error) { n := &NetConf{} - if err := json.Unmarshal(bytes, n); err != nil { + var err error + if err = json.Unmarshal(bytes, n); err != nil { return nil, fmt.Errorf("failed to load netconf: %v", err) } @@ -80,6 +82,13 @@ func loadConf(bytes []byte) (*NetConf, error) { return nil, fmt.Errorf(`specify either "device", "hwaddr", "kernelpath" or "pciBusID"`) } + if len(n.PCIAddr) > 0 { + n.DPDKMode, err = hasDpdkDriver(n.PCIAddr) + if err != nil { + return nil, fmt.Errorf("error with host device: %v", err) + } + } + return n, nil } @@ -94,49 +103,17 @@ func cmdAdd(args *skel.CmdArgs) error { } defer containerNs.Close() - if len(cfg.PCIAddr) > 0 { - isDpdkMode, err := hasDpdkDriver(cfg.PCIAddr) + result := ¤t.Result{} + var contDev netlink.Link + if !cfg.DPDKMode { + hostDev, err := getLink(cfg.Device, cfg.HWAddr, cfg.KernelPath, cfg.PCIAddr) if err != nil { - return fmt.Errorf("error with host device: %v", err) + return fmt.Errorf("failed to find host device: %v", err) } - if isDpdkMode { - return types.PrintResult(¤t.Result{}, cfg.CNIVersion) - } - } - hostDev, err := getLink(cfg.Device, cfg.HWAddr, cfg.KernelPath, cfg.PCIAddr) - if err != nil { - return fmt.Errorf("failed to find host device: %v", err) - } - - contDev, err := moveLinkIn(hostDev, containerNs, args.IfName) - if err != nil { - return fmt.Errorf("failed to move link %v", err) - } - - var result *current.Result - // run the IPAM plugin and get back the config to apply - if cfg.IPAM.Type != "" { - r, err := ipam.ExecAdd(cfg.IPAM.Type, args.StdinData) + contDev, err = moveLinkIn(hostDev, containerNs, args.IfName) if err != nil { - return err - } - - // Invoke ipam del if err to avoid ip leak - defer func() { - if err != nil { - ipam.ExecDel(cfg.IPAM.Type, args.StdinData) - } - }() - - // Convert whatever the IPAM result was into the current Result type - result, err = current.NewResultFromResult(r) - if err != nil { - return err - } - - if len(result.IPs) == 0 { - return errors.New("IPAM plugin returned missing IP config") + return fmt.Errorf("failed to move link %v", err) } result.Interfaces = []*current.Interface{{ @@ -144,13 +121,48 @@ func cmdAdd(args *skel.CmdArgs) error { Mac: contDev.Attrs().HardwareAddr.String(), Sandbox: containerNs.Path(), }} - for _, ipc := range result.IPs { - // All addresses apply to the container interface (move from host) - ipc.Interface = current.Int(0) - } + } + if cfg.IPAM.Type == "" { + if cfg.DPDKMode { + return types.PrintResult(result, cfg.CNIVersion) + } + return printLink(contDev, cfg.CNIVersion, containerNs) + } + + // run the IPAM plugin and get back the config to apply + r, err := ipam.ExecAdd(cfg.IPAM.Type, args.StdinData) + if err != nil { + return err + } + + // Invoke ipam del if err to avoid ip leak + defer func() { + if err != nil { + ipam.ExecDel(cfg.IPAM.Type, args.StdinData) + } + }() + + // Convert whatever the IPAM result was into the current Result type + newResult, err := current.NewResultFromResult(r) + if err != nil { + return err + } + + if len(newResult.IPs) == 0 { + return errors.New("IPAM plugin returned missing IP config") + } + + for _, ipc := range newResult.IPs { + // All addresses apply to the container interface (move from host) + ipc.Interface = current.Int(0) + } + + newResult.Interfaces = result.Interfaces + + if !cfg.DPDKMode { err = containerNs.Do(func(_ ns.NetNS) error { - if err := ipam.ConfigureIface(args.IfName, result); err != nil { + if err := ipam.ConfigureIface(args.IfName, newResult); err != nil { return err } return nil @@ -158,13 +170,11 @@ func cmdAdd(args *skel.CmdArgs) error { if err != nil { return err } - - result.DNS = cfg.DNS - - return types.PrintResult(result, cfg.CNIVersion) } - return printLink(contDev, cfg.CNIVersion, containerNs) + newResult.DNS = cfg.DNS + + return types.PrintResult(newResult, cfg.CNIVersion) } func cmdDel(args *skel.CmdArgs) error { @@ -181,26 +191,18 @@ func cmdDel(args *skel.CmdArgs) error { } defer containerNs.Close() - if len(cfg.PCIAddr) > 0 { - isDpdkMode, err := hasDpdkDriver(cfg.PCIAddr) - if err != nil { - return fmt.Errorf("error with host device: %v", err) - } - if isDpdkMode { - return nil - } - } - - if err := moveLinkOut(containerNs, args.IfName); err != nil { - return err - } - if cfg.IPAM.Type != "" { if err := ipam.ExecDel(cfg.IPAM.Type, args.StdinData); err != nil { return err } } + if !cfg.DPDKMode { + if err := moveLinkOut(containerNs, args.IfName); err != nil { + return err + } + } + return nil } @@ -410,6 +412,10 @@ func cmdCheck(args *skel.CmdArgs) error { return err } + if cfg.DPDKMode { + return nil + } + var contMap current.Interface // Find interfaces for name we know, that of host-device inside container for _, intf := range result.Interfaces { From 547a516c30f5da47f1915fb2a88e700db70ed4f5 Mon Sep 17 00:00:00 2001 From: Periyasamy Palanisamy Date: Thu, 18 Nov 2021 10:04:11 +0100 Subject: [PATCH 2/2] add ipam tests for dpdk device Signed-off-by: Periyasamy Palanisamy --- plugins/main/host-device/host-device.go | 2 +- plugins/main/host-device/host-device_test.go | 216 ++++++++++++++++++- 2 files changed, 215 insertions(+), 3 deletions(-) diff --git a/plugins/main/host-device/host-device.go b/plugins/main/host-device/host-device.go index f6fc35c7..eda1412d 100644 --- a/plugins/main/host-device/host-device.go +++ b/plugins/main/host-device/host-device.go @@ -39,7 +39,7 @@ import ( bv "github.com/containernetworking/plugins/pkg/utils/buildversion" ) -const ( +var ( sysBusPCI = "/sys/bus/pci/devices" ) diff --git a/plugins/main/host-device/host-device_test.go b/plugins/main/host-device/host-device_test.go index 0a7b7c02..057a8560 100644 --- a/plugins/main/host-device/host-device_test.go +++ b/plugins/main/host-device/host-device_test.go @@ -17,14 +17,17 @@ package main import ( "encoding/json" "fmt" + "io/ioutil" "math/rand" "net" + "os" + "path" "strings" "github.com/containernetworking/cni/pkg/skel" "github.com/containernetworking/cni/pkg/types" - "github.com/containernetworking/cni/pkg/types/040" - "github.com/containernetworking/cni/pkg/types/100" + types040 "github.com/containernetworking/cni/pkg/types/040" + types100 "github.com/containernetworking/cni/pkg/types/100" "github.com/containernetworking/cni/pkg/version" "github.com/containernetworking/plugins/pkg/ns" "github.com/containernetworking/plugins/pkg/testutils" @@ -217,6 +220,7 @@ func buildOneConfig(name, cniVersion string, orig *Net, prevResult types.Result) type tester interface { expectInterfaces(result types.Result, name, mac, sandbox string) + expectDpdkInterfaceIP(result types.Result, ipAddress string) } type testerBase struct{} @@ -252,6 +256,15 @@ func (t *testerV10x) expectInterfaces(result types.Result, name, mac, sandbox st })) } +func (t *testerV10x) expectDpdkInterfaceIP(result types.Result, ipAddress string) { + // check that the result was sane + res, err := types100.NewResultFromResult(result) + Expect(err).NotTo(HaveOccurred()) + Expect(len(res.Interfaces)).To(Equal(0)) + Expect(len(res.IPs)).To(Equal(1)) + Expect(res.IPs[0].Address.String()).To(Equal(ipAddress)) +} + func (t *testerV04x) expectInterfaces(result types.Result, name, mac, sandbox string) { // check that the result was sane res, err := types040.NewResultFromResult(result) @@ -265,6 +278,15 @@ func (t *testerV04x) expectInterfaces(result types.Result, name, mac, sandbox st })) } +func (t *testerV04x) expectDpdkInterfaceIP(result types.Result, ipAddress string) { + // check that the result was sane + res, err := types040.NewResultFromResult(result) + Expect(err).NotTo(HaveOccurred()) + Expect(len(res.Interfaces)).To(Equal(0)) + Expect(len(res.IPs)).To(Equal(1)) + Expect(res.IPs[0].Address.String()).To(Equal(ipAddress)) +} + func (t *testerV03x) expectInterfaces(result types.Result, name, mac, sandbox string) { // check that the result was sane res, err := types040.NewResultFromResult(result) @@ -278,6 +300,15 @@ func (t *testerV03x) expectInterfaces(result types.Result, name, mac, sandbox st })) } +func (t *testerV03x) expectDpdkInterfaceIP(result types.Result, ipAddress string) { + // check that the result was sane + res, err := types040.NewResultFromResult(result) + Expect(err).NotTo(HaveOccurred()) + Expect(len(res.Interfaces)).To(Equal(0)) + Expect(len(res.IPs)).To(Equal(1)) + Expect(res.IPs[0].Address.String()).To(Equal(ipAddress)) +} + var _ = Describe("base functionality", func() { var originalNS, targetNS ns.NetNS var ifname string @@ -509,6 +540,65 @@ var _ = Describe("base functionality", func() { }) }) + It(fmt.Sprintf("Works with a valid %s config on a DPDK device with IPAM", ver), func() { + fs := &fakeFilesystem{ + dirs: []string{ + "sys/bus/pci/devices/0000:00:00.1", + "sys/bus/pci/drivers/vfio-pci", + }, + symlinks: map[string]string{ + "sys/bus/pci/devices/0000:00:00.1/driver": "../../../../bus/pci/drivers/vfio-pci", + }, + } + defer fs.use()() + + // call CmdAdd + targetIP := "10.10.0.1/24" + cniName := "eth0" + conf := fmt.Sprintf(`{ + "cniVersion": "%s", + "name": "cni-plugin-host-device-test", + "type": "host-device", + "ipam": { + "type": "static", + "addresses": [ + { + "address":"`+targetIP+`", + "gateway": "10.10.0.254" + }] + }, + "pciBusID": %q + }`, ver, "0000:00:00.1") + args := &skel.CmdArgs{ + ContainerID: "dummy", + IfName: cniName, + Netns: targetNS.Path(), + StdinData: []byte(conf), + } + var resI types.Result + err := originalNS.Do(func(ns.NetNS) error { + defer GinkgoRecover() + var err error + resI, _, err = testutils.CmdAddWithArgs(args, func() error { return cmdAdd(args) }) + return err + }) + Expect(err).NotTo(HaveOccurred()) + + // check that the result was sane + t := newTesterByVersion(ver) + t.expectDpdkInterfaceIP(resI, targetIP) + + // call CmdDel + _ = originalNS.Do(func(ns.NetNS) error { + defer GinkgoRecover() + err = testutils.CmdDelWithArgs(args, func() error { + return cmdDel(args) + }) + Expect(err).NotTo(HaveOccurred()) + return nil + }) + }) + It(fmt.Sprintf("Works with a valid %s config with IPAM", ver), func() { var origLink netlink.Link @@ -721,6 +811,89 @@ var _ = Describe("base functionality", func() { }) }) + It(fmt.Sprintf("Works with a valid %s config on a DPDK device with IPAM", ver), func() { + fs := &fakeFilesystem{ + dirs: []string{ + "sys/bus/pci/devices/0000:00:00.1", + "sys/bus/pci/drivers/vfio-pci", + }, + symlinks: map[string]string{ + "sys/bus/pci/devices/0000:00:00.1/driver": "../../../../bus/pci/drivers/vfio-pci", + }, + } + defer fs.use()() + + // call CmdAdd + targetIP := "10.10.0.1/24" + cniName := "eth0" + conf := fmt.Sprintf(`{ + "cniVersion": "%s", + "name": "cni-plugin-host-device-test", + "type": "host-device", + "ipam": { + "type": "static", + "addresses": [ + { + "address":"`+targetIP+`", + "gateway": "10.10.0.254" + }] + }, + "pciBusID": %q + }`, ver, "0000:00:00.1") + args := &skel.CmdArgs{ + ContainerID: "dummy", + Netns: targetNS.Path(), + IfName: cniName, + StdinData: []byte(conf), + } + var resI types.Result + err := originalNS.Do(func(ns.NetNS) error { + defer GinkgoRecover() + var err error + resI, _, err = testutils.CmdAddWithArgs(args, func() error { return cmdAdd(args) }) + return err + }) + Expect(err).NotTo(HaveOccurred()) + + // check that the result was sane + t := newTesterByVersion(ver) + t.expectDpdkInterfaceIP(resI, targetIP) + + // call CmdCheck + n := &Net{} + err = json.Unmarshal([]byte(conf), &n) + Expect(err).NotTo(HaveOccurred()) + + n.IPAM, _, err = LoadIPAMConfig([]byte(conf), "") + Expect(err).NotTo(HaveOccurred()) + + if testutils.SpecVersionHasCHECK(ver) { + newConf, err := buildOneConfig("testConfig", ver, n, resI) + Expect(err).NotTo(HaveOccurred()) + + confString, err := json.Marshal(newConf) + Expect(err).NotTo(HaveOccurred()) + + args.StdinData = confString + + err = originalNS.Do(func(ns.NetNS) error { + defer GinkgoRecover() + return testutils.CmdCheckWithArgs(args, func() error { return cmdCheck(args) }) + }) + Expect(err).NotTo(HaveOccurred()) + } + + // call CmdDel + _ = originalNS.Do(func(ns.NetNS) error { + defer GinkgoRecover() + err = testutils.CmdDelWithArgs(args, func() error { + return cmdDel(args) + }) + Expect(err).NotTo(HaveOccurred()) + return nil + }) + }) + It(fmt.Sprintf("Works with a valid %s config with IPAM", ver), func() { var origLink netlink.Link @@ -971,3 +1144,42 @@ var _ = Describe("base functionality", func() { }) } }) + +type fakeFilesystem struct { + rootDir string + dirs []string + symlinks map[string]string +} + +func (fs *fakeFilesystem) use() func() { + // create the new fake fs root dir in /tmp/sriov... + tmpDir, err := ioutil.TempDir("", "sriov") + if err != nil { + panic(fmt.Errorf("error creating fake root dir: %s", err.Error())) + } + fs.rootDir = tmpDir + + for _, dir := range fs.dirs { + err := os.MkdirAll(path.Join(fs.rootDir, dir), 0755) + if err != nil { + panic(fmt.Errorf("error creating fake directory: %s", err.Error())) + } + } + + for link, target := range fs.symlinks { + err = os.Symlink(target, path.Join(fs.rootDir, link)) + if err != nil { + panic(fmt.Errorf("error creating fake symlink: %s", err.Error())) + } + } + + sysBusPCI = path.Join(fs.rootDir, "/sys/bus/pci/devices") + + return func() { + // remove temporary fake fs + err := os.RemoveAll(fs.rootDir) + if err != nil { + panic(fmt.Errorf("error tearing down fake filesystem: %s", err.Error())) + } + } +}