Merge pull request #642 from Nordix/dpdk-ipam

host-device: add ipam support for dpdk device

plugins/main/host-device/host-device.go

@@ -39,7 +39,7 @@ import (
 	bv "github.com/containernetworking/plugins/pkg/utils/buildversion"
 )
 
-const (
+var (
 	sysBusPCI = "/sys/bus/pci/devices"
 )
 
@@ -51,6 +51,7 @@ type NetConf struct {
 	types.NetConf
 	Device        string `json:"device"` // Device-Name, something like eth0 or can0 etc.
 	HWAddr        string `json:"hwaddr"` // MAC Address of target network interface
+	DPDKMode      bool
 	KernelPath    string `json:"kernelpath"` // Kernelpath of the device
 	PCIAddr       string `json:"pciBusID"`   // PCI Address of target network device
 	RuntimeConfig struct {
@@ -67,7 +68,8 @@ func init() {
 
 func loadConf(bytes []byte) (*NetConf, error) {
 	n := &NetConf{}
-	if err := json.Unmarshal(bytes, n); err != nil {
+	var err error
+	if err = json.Unmarshal(bytes, n); err != nil {
 		return nil, fmt.Errorf("failed to load netconf: %v", err)
 	}
 
@@ -80,6 +82,13 @@ func loadConf(bytes []byte) (*NetConf, error) {
 		return nil, fmt.Errorf(`specify either "device", "hwaddr", "kernelpath" or "pciBusID"`)
 	}
 
+	if len(n.PCIAddr) > 0 {
+		n.DPDKMode, err = hasDpdkDriver(n.PCIAddr)
+		if err != nil {
+			return nil, fmt.Errorf("error with host device: %v", err)
+		}
+	}
+
 	return n, nil
 }
 
@@ -94,29 +103,34 @@ func cmdAdd(args *skel.CmdArgs) error {
 	}
 	defer containerNs.Close()
 
-	if len(cfg.PCIAddr) > 0 {
+	result := &current.Result{}
-		isDpdkMode, err := hasDpdkDriver(cfg.PCIAddr)
+	var contDev netlink.Link
-		if err != nil {
+	if !cfg.DPDKMode {
-			return fmt.Errorf("error with host device: %v", err)
-		}
-		if isDpdkMode {
-			return types.PrintResult(&current.Result{}, cfg.CNIVersion)
-		}
-	}
-
 		hostDev, err := getLink(cfg.Device, cfg.HWAddr, cfg.KernelPath, cfg.PCIAddr)
 		if err != nil {
 			return fmt.Errorf("failed to find host device: %v", err)
 		}
 
-	contDev, err := moveLinkIn(hostDev, containerNs, args.IfName)
+		contDev, err = moveLinkIn(hostDev, containerNs, args.IfName)
 		if err != nil {
 			return fmt.Errorf("failed to move link %v", err)
 		}
 
-	var result *current.Result
+		result.Interfaces = []*current.Interface{{
+			Name:    contDev.Attrs().Name,
+			Mac:     contDev.Attrs().HardwareAddr.String(),
+			Sandbox: containerNs.Path(),
+		}}
+	}
+
+	if cfg.IPAM.Type == "" {
+		if cfg.DPDKMode {
+			return types.PrintResult(result, cfg.CNIVersion)
+		}
+		return printLink(contDev, cfg.CNIVersion, containerNs)
+	}
+
 	// run the IPAM plugin and get back the config to apply
-	if cfg.IPAM.Type != "" {
 	r, err := ipam.ExecAdd(cfg.IPAM.Type, args.StdinData)
 	if err != nil {
 		return err
@@ -130,27 +144,25 @@ func cmdAdd(args *skel.CmdArgs) error {
 	}()
 
 	// Convert whatever the IPAM result was into the current Result type
-		result, err = current.NewResultFromResult(r)
+	newResult, err := current.NewResultFromResult(r)
 	if err != nil {
 		return err
 	}
 
-		if len(result.IPs) == 0 {
+	if len(newResult.IPs) == 0 {
 		return errors.New("IPAM plugin returned missing IP config")
 	}
 
-		result.Interfaces = []*current.Interface{{
+	for _, ipc := range newResult.IPs {
-			Name:    contDev.Attrs().Name,
-			Mac:     contDev.Attrs().HardwareAddr.String(),
-			Sandbox: containerNs.Path(),
-		}}
-		for _, ipc := range result.IPs {
 		// All addresses apply to the container interface (move from host)
 		ipc.Interface = current.Int(0)
 	}
 
+	newResult.Interfaces = result.Interfaces
+
+	if !cfg.DPDKMode {
 		err = containerNs.Do(func(_ ns.NetNS) error {
-			if err := ipam.ConfigureIface(args.IfName, result); err != nil {
+			if err := ipam.ConfigureIface(args.IfName, newResult); err != nil {
 				return err
 			}
 			return nil
@@ -158,13 +170,11 @@ func cmdAdd(args *skel.CmdArgs) error {
 		if err != nil {
 			return err
 		}
-
-		result.DNS = cfg.DNS
-
-		return types.PrintResult(result, cfg.CNIVersion)
 	}
 
-	return printLink(contDev, cfg.CNIVersion, containerNs)
+	newResult.DNS = cfg.DNS
+
+	return types.PrintResult(newResult, cfg.CNIVersion)
 }
 
 func cmdDel(args *skel.CmdArgs) error {
@@ -181,26 +191,18 @@ func cmdDel(args *skel.CmdArgs) error {
 	}
 	defer containerNs.Close()
 
-	if len(cfg.PCIAddr) > 0 {
-		isDpdkMode, err := hasDpdkDriver(cfg.PCIAddr)
-		if err != nil {
-			return fmt.Errorf("error with host device: %v", err)
-		}
-		if isDpdkMode {
-			return nil
-		}
-	}
-
-	if err := moveLinkOut(containerNs, args.IfName); err != nil {
-		return err
-	}
-
 	if cfg.IPAM.Type != "" {
 		if err := ipam.ExecDel(cfg.IPAM.Type, args.StdinData); err != nil {
 			return err
 		}
 	}
 
+	if !cfg.DPDKMode {
+		if err := moveLinkOut(containerNs, args.IfName); err != nil {
+			return err
+		}
+	}
+
 	return nil
 }
 
@@ -410,6 +412,10 @@ func cmdCheck(args *skel.CmdArgs) error {
 		return err
 	}
 
+	if cfg.DPDKMode {
+		return nil
+	}
+
 	var contMap current.Interface
 	// Find interfaces for name we know, that of host-device inside container
 	for _, intf := range result.Interfaces {

plugins/main/host-device/host-device_test.go

@@ -17,14 +17,17 @@ package main
 import (
 	"encoding/json"
 	"fmt"
+	"io/ioutil"
 	"math/rand"
 	"net"
+	"os"
+	"path"
 	"strings"
 
 	"github.com/containernetworking/cni/pkg/skel"
 	"github.com/containernetworking/cni/pkg/types"
-	"github.com/containernetworking/cni/pkg/types/040"
+	types040 "github.com/containernetworking/cni/pkg/types/040"
-	"github.com/containernetworking/cni/pkg/types/100"
+	types100 "github.com/containernetworking/cni/pkg/types/100"
 	"github.com/containernetworking/cni/pkg/version"
 	"github.com/containernetworking/plugins/pkg/ns"
 	"github.com/containernetworking/plugins/pkg/testutils"
@@ -217,6 +220,7 @@ func buildOneConfig(name, cniVersion string, orig *Net, prevResult types.Result)
 
 type tester interface {
 	expectInterfaces(result types.Result, name, mac, sandbox string)
+	expectDpdkInterfaceIP(result types.Result, ipAddress string)
 }
 
 type testerBase struct{}
@@ -252,6 +256,15 @@ func (t *testerV10x) expectInterfaces(result types.Result, name, mac, sandbox st
 	}))
 }
 
+func (t *testerV10x) expectDpdkInterfaceIP(result types.Result, ipAddress string) {
+	// check that the result was sane
+	res, err := types100.NewResultFromResult(result)
+	Expect(err).NotTo(HaveOccurred())
+	Expect(len(res.Interfaces)).To(Equal(0))
+	Expect(len(res.IPs)).To(Equal(1))
+	Expect(res.IPs[0].Address.String()).To(Equal(ipAddress))
+}
+
 func (t *testerV04x) expectInterfaces(result types.Result, name, mac, sandbox string) {
 	// check that the result was sane
 	res, err := types040.NewResultFromResult(result)
@@ -265,6 +278,15 @@ func (t *testerV04x) expectInterfaces(result types.Result, name, mac, sandbox st
 	}))
 }
 
+func (t *testerV04x) expectDpdkInterfaceIP(result types.Result, ipAddress string) {
+	// check that the result was sane
+	res, err := types040.NewResultFromResult(result)
+	Expect(err).NotTo(HaveOccurred())
+	Expect(len(res.Interfaces)).To(Equal(0))
+	Expect(len(res.IPs)).To(Equal(1))
+	Expect(res.IPs[0].Address.String()).To(Equal(ipAddress))
+}
+
 func (t *testerV03x) expectInterfaces(result types.Result, name, mac, sandbox string) {
 	// check that the result was sane
 	res, err := types040.NewResultFromResult(result)
@@ -278,6 +300,15 @@ func (t *testerV03x) expectInterfaces(result types.Result, name, mac, sandbox st
 	}))
 }
 
+func (t *testerV03x) expectDpdkInterfaceIP(result types.Result, ipAddress string) {
+	// check that the result was sane
+	res, err := types040.NewResultFromResult(result)
+	Expect(err).NotTo(HaveOccurred())
+	Expect(len(res.Interfaces)).To(Equal(0))
+	Expect(len(res.IPs)).To(Equal(1))
+	Expect(res.IPs[0].Address.String()).To(Equal(ipAddress))
+}
+
 var _ = Describe("base functionality", func() {
 	var originalNS, targetNS ns.NetNS
 	var ifname string
@@ -509,6 +540,65 @@ var _ = Describe("base functionality", func() {
 			})
 		})
 
+		It(fmt.Sprintf("Works with a valid %s config on a DPDK device with IPAM", ver), func() {
+			fs := &fakeFilesystem{
+				dirs: []string{
+					"sys/bus/pci/devices/0000:00:00.1",
+					"sys/bus/pci/drivers/vfio-pci",
+				},
+				symlinks: map[string]string{
+					"sys/bus/pci/devices/0000:00:00.1/driver": "../../../../bus/pci/drivers/vfio-pci",
+				},
+			}
+			defer fs.use()()
+
+			// call CmdAdd
+			targetIP := "10.10.0.1/24"
+			cniName := "eth0"
+			conf := fmt.Sprintf(`{
+							"cniVersion": "%s",
+							"name": "cni-plugin-host-device-test",
+							"type": "host-device",
+							"ipam": {
+								"type": "static",
+								"addresses": [
+									{
+									"address":"`+targetIP+`",
+									"gateway": "10.10.0.254"
+								}]
+							},
+							"pciBusID": %q
+						}`, ver, "0000:00:00.1")
+			args := &skel.CmdArgs{
+				ContainerID: "dummy",
+				IfName:      cniName,
+				Netns:       targetNS.Path(),
+				StdinData:   []byte(conf),
+			}
+			var resI types.Result
+			err := originalNS.Do(func(ns.NetNS) error {
+				defer GinkgoRecover()
+				var err error
+				resI, _, err = testutils.CmdAddWithArgs(args, func() error { return cmdAdd(args) })
+				return err
+			})
+			Expect(err).NotTo(HaveOccurred())
+
+			// check that the result was sane
+			t := newTesterByVersion(ver)
+			t.expectDpdkInterfaceIP(resI, targetIP)
+
+			// call CmdDel
+			_ = originalNS.Do(func(ns.NetNS) error {
+				defer GinkgoRecover()
+				err = testutils.CmdDelWithArgs(args, func() error {
+					return cmdDel(args)
+				})
+				Expect(err).NotTo(HaveOccurred())
+				return nil
+			})
+		})
+
 		It(fmt.Sprintf("Works with a valid %s config with IPAM", ver), func() {
 			var origLink netlink.Link
 
@@ -721,6 +811,89 @@ var _ = Describe("base functionality", func() {
 			})
 		})
 
+		It(fmt.Sprintf("Works with a valid %s config on a DPDK device with IPAM", ver), func() {
+			fs := &fakeFilesystem{
+				dirs: []string{
+					"sys/bus/pci/devices/0000:00:00.1",
+					"sys/bus/pci/drivers/vfio-pci",
+				},
+				symlinks: map[string]string{
+					"sys/bus/pci/devices/0000:00:00.1/driver": "../../../../bus/pci/drivers/vfio-pci",
+				},
+			}
+			defer fs.use()()
+
+			// call CmdAdd
+			targetIP := "10.10.0.1/24"
+			cniName := "eth0"
+			conf := fmt.Sprintf(`{
+							"cniVersion": "%s",
+							"name": "cni-plugin-host-device-test",
+							"type": "host-device",
+							"ipam": {
+								"type": "static",
+								"addresses": [
+									{
+									"address":"`+targetIP+`",
+									"gateway": "10.10.0.254"
+								}]
+							},
+							"pciBusID": %q
+						}`, ver, "0000:00:00.1")
+			args := &skel.CmdArgs{
+				ContainerID: "dummy",
+				Netns:       targetNS.Path(),
+				IfName:      cniName,
+				StdinData:   []byte(conf),
+			}
+			var resI types.Result
+			err := originalNS.Do(func(ns.NetNS) error {
+				defer GinkgoRecover()
+				var err error
+				resI, _, err = testutils.CmdAddWithArgs(args, func() error { return cmdAdd(args) })
+				return err
+			})
+			Expect(err).NotTo(HaveOccurred())
+
+			// check that the result was sane
+			t := newTesterByVersion(ver)
+			t.expectDpdkInterfaceIP(resI, targetIP)
+
+			// call CmdCheck
+			n := &Net{}
+			err = json.Unmarshal([]byte(conf), &n)
+			Expect(err).NotTo(HaveOccurred())
+
+			n.IPAM, _, err = LoadIPAMConfig([]byte(conf), "")
+			Expect(err).NotTo(HaveOccurred())
+
+			if testutils.SpecVersionHasCHECK(ver) {
+				newConf, err := buildOneConfig("testConfig", ver, n, resI)
+				Expect(err).NotTo(HaveOccurred())
+
+				confString, err := json.Marshal(newConf)
+				Expect(err).NotTo(HaveOccurred())
+
+				args.StdinData = confString
+
+				err = originalNS.Do(func(ns.NetNS) error {
+					defer GinkgoRecover()
+					return testutils.CmdCheckWithArgs(args, func() error { return cmdCheck(args) })
+				})
+				Expect(err).NotTo(HaveOccurred())
+			}
+
+			// call CmdDel
+			_ = originalNS.Do(func(ns.NetNS) error {
+				defer GinkgoRecover()
+				err = testutils.CmdDelWithArgs(args, func() error {
+					return cmdDel(args)
+				})
+				Expect(err).NotTo(HaveOccurred())
+				return nil
+			})
+		})
+
 		It(fmt.Sprintf("Works with a valid %s config with IPAM", ver), func() {
 			var origLink netlink.Link
 
@@ -971,3 +1144,42 @@ var _ = Describe("base functionality", func() {
 		})
 	}
 })
+
+type fakeFilesystem struct {
+	rootDir  string
+	dirs     []string
+	symlinks map[string]string
+}
+
+func (fs *fakeFilesystem) use() func() {
+	// create the new fake fs root dir in /tmp/sriov...
+	tmpDir, err := ioutil.TempDir("", "sriov")
+	if err != nil {
+		panic(fmt.Errorf("error creating fake root dir: %s", err.Error()))
+	}
+	fs.rootDir = tmpDir
+
+	for _, dir := range fs.dirs {
+		err := os.MkdirAll(path.Join(fs.rootDir, dir), 0755)
+		if err != nil {
+			panic(fmt.Errorf("error creating fake directory: %s", err.Error()))
+		}
+	}
+
+	for link, target := range fs.symlinks {
+		err = os.Symlink(target, path.Join(fs.rootDir, link))
+		if err != nil {
+			panic(fmt.Errorf("error creating fake symlink: %s", err.Error()))
+		}
+	}
+
+	sysBusPCI = path.Join(fs.rootDir, "/sys/bus/pci/devices")
+
+	return func() {
+		// remove temporary fake fs
+		err := os.RemoveAll(fs.rootDir)
+		if err != nil {
+			panic(fmt.Errorf("error tearing down fake filesystem: %s", err.Error()))
+		}
+	}
+}