ITQC/containernetworking-plugins
24
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 34 Wiki Activity

spec/plugins: return interface details and multiple IP addresses to runtime

Browse Source 
Updates the spec and plugins to return an array of interfaces and IP details
to the runtime including:

- interface names and MAC addresses configured by the plugin
- whether the interfaces are sandboxed (container/VM) or host (bridge, veth, etc)
- multiple IP addresses configured by IPAM and which interface they
have been assigned to

Returning interface details is useful for runtimes, as well as allowing
more flexible chaining of CNI plugins themselves.  For example, some
meta plugins may need to know the host-side interface to be able to
apply firewall or traffic shaping rules to the container.
This commit is contained in:
Dan Williams
2016-11-22 11:32:35 -06:00
parent befad17174
commit d5acb127b8
40 changed files with 1653 additions and 499 deletions
Download Patch File Download Diff File Expand all files Collapse all files

185
plugins/main/bridge/bridge.go
View File

@ -53,14 +53,14 @@ func init() {
runtime.LockOSThread()
}
func loadNetConf(bytes []byte) (*NetConf, error) {
func loadNetConf(bytes []byte) (*NetConf, string, error) {
n := &NetConf{
BrName: defaultBrName,
}
if err := json.Unmarshal(bytes, n); err != nil {
return nil, fmt.Errorf("failed to load netconf: %v", err)
return nil, "", fmt.Errorf("failed to load netconf: %v", err)
}
return n, nil
return n, n.CNIVersion, nil
}
func ensureBridgeAddr(br *netlink.Bridge, ipn *net.IPNet, forceAddress bool) error {
@ -139,16 +139,16 @@ func ensureBridge(brName string, mtu int) (*netlink.Bridge, error) {
},
}
if err := netlink.LinkAdd(br); err != nil {
if err != syscall.EEXIST {
return nil, fmt.Errorf("could not add %q: %v", brName, err)
}
err := netlink.LinkAdd(br)
if err != nil && err != syscall.EEXIST {
return nil, fmt.Errorf("could not add %q: %v", brName, err)
}
// it's ok if the device already exists as long as config is similar
br, err = bridgeByName(brName)
if err != nil {
return nil, err
}
// Re-fetch link to read all attributes and if it already existed,
// ensure it's really a bridge with similar configuration
br, err = bridgeByName(brName)
if err != nil {
return nil, err
}
if err := netlink.LinkSetUp(br); err != nil {
@ -158,40 +158,44 @@ func ensureBridge(brName string, mtu int) (*netlink.Bridge, error) {
return br, nil
}
func setupVeth(netns ns.NetNS, br *netlink.Bridge, ifName string, mtu int, hairpinMode bool) error {
var hostVethName string
func setupVeth(netns ns.NetNS, br *netlink.Bridge, ifName string, mtu int, hairpinMode bool) (*current.Interface, *current.Interface, error) {
contIface := &current.Interface{}
hostIface := &current.Interface{}
err := netns.Do(func(hostNS ns.NetNS) error {
// create the veth pair in the container and move host end into host netns
hostVeth, _, err := ip.SetupVeth(ifName, mtu, hostNS)
hostVeth, containerVeth, err := ip.SetupVeth(ifName, mtu, hostNS)
if err != nil {
return err
}
hostVethName = hostVeth.Attrs().Name
contIface.Name = containerVeth.Attrs().Name
contIface.Mac = containerVeth.Attrs().HardwareAddr.String()
contIface.Sandbox = netns.Path()
hostIface.Name = hostVeth.Attrs().Name
return nil
})
if err != nil {
return err
return nil, nil, err
}
// need to lookup hostVeth again as its index has changed during ns move
hostVeth, err := netlink.LinkByName(hostVethName)
hostVeth, err := netlink.LinkByName(hostIface.Name)
if err != nil {
return fmt.Errorf("failed to lookup %q: %v", hostVethName, err)
return nil, nil, fmt.Errorf("failed to lookup %q: %v", hostIface.Name, err)
}
hostIface.Mac = hostVeth.Attrs().HardwareAddr.String()
// connect host veth end to the bridge
if err = netlink.LinkSetMaster(hostVeth, br); err != nil {
return fmt.Errorf("failed to connect %q to bridge %v: %v", hostVethName, br.Attrs().Name, err)
if err := netlink.LinkSetMaster(hostVeth, br); err != nil {
return nil, nil, fmt.Errorf("failed to connect %q to bridge %v: %v", hostVeth.Attrs().Name, br.Attrs().Name, err)
}
// set hairpin mode
if err = netlink.LinkSetHairpin(hostVeth, hairpinMode); err != nil {
return fmt.Errorf("failed to setup hairpin mode for %v: %v", hostVethName, err)
return nil, nil, fmt.Errorf("failed to setup hairpin mode for %v: %v", hostVeth.Attrs().Name, err)
}
return nil
return hostIface, contIface, nil
}
func calcGatewayIP(ipn *net.IPNet) net.IP {
@ -199,18 +203,21 @@ func calcGatewayIP(ipn *net.IPNet) net.IP {
return ip.NextIP(nid)
}
func setupBridge(n *NetConf) (*netlink.Bridge, error) {
func setupBridge(n *NetConf) (*netlink.Bridge, *current.Interface, error) {
// create bridge if necessary
br, err := ensureBridge(n.BrName, n.MTU)
if err != nil {
return nil, fmt.Errorf("failed to create bridge %q: %v", n.BrName, err)
return nil, nil, fmt.Errorf("failed to create bridge %q: %v", n.BrName, err)
}
return br, nil
return br, &current.Interface{
Name: br.Attrs().Name,
Mac: br.Attrs().HardwareAddr.String(),
}, nil
}
func cmdAdd(args *skel.CmdArgs) error {
n, err := loadNetConf(args.StdinData)
n, cniVersion, err := loadNetConf(args.StdinData)
if err != nil {
return err
}
@ -219,7 +226,7 @@ func cmdAdd(args *skel.CmdArgs) error {
n.IsGW = true
}
br, err := setupBridge(n)
br, brInterface, err := setupBridge(n)
if err != nil {
return err
}
@ -230,7 +237,8 @@ func cmdAdd(args *skel.CmdArgs) error {
}
defer netns.Close()
if err = setupVeth(netns, br, args.IfName, n.MTU, n.HairpinMode); err != nil {
hostInterface, containerInterface, err := setupVeth(netns, br, args.IfName, n.MTU, n.HairpinMode)
if err != nil {
return err
}
@ -240,72 +248,100 @@ func cmdAdd(args *skel.CmdArgs) error {
return err
}
result, err := current.GetResult(r)
// Convert whatever the IPAM result was into the current Result type
result, err := current.NewResultFromResult(r)
if err != nil {
return err
}
// TODO: make this optional when IPv6 is supported
if result.IP4 == nil {
return errors.New("IPAM plugin returned missing IPv4 config")
if len(result.IPs) == 0 {
return errors.New("IPAM plugin returned missing IP config")
}
if result.IP4.Gateway == nil && n.IsGW {
result.IP4.Gateway = calcGatewayIP(&result.IP4.IP)
result.Interfaces = []*current.Interface{brInterface, hostInterface, containerInterface}
for _, ipc := range result.IPs {
// All IPs currently refer to the container interface
ipc.Interface = 2
if ipc.Gateway == nil && n.IsGW {
ipc.Gateway = calcGatewayIP(&ipc.Address)
}
}
if err := netns.Do(func(_ ns.NetNS) error {
// set the default gateway if requested
if n.IsDefaultGW {
_, defaultNet, err := net.ParseCIDR("0.0.0.0/0")
if err != nil {
return err
}
for _, ipc := range result.IPs {
defaultNet := &net.IPNet{}
switch {
case ipc.Address.IP.To4() != nil:
defaultNet.IP = net.IPv4zero
defaultNet.Mask = net.IPMask(net.IPv4zero)
case len(ipc.Address.IP) == net.IPv6len && ipc.Address.IP.To4() == nil:
defaultNet.IP = net.IPv6zero
defaultNet.Mask = net.IPMask(net.IPv6zero)
default:
return fmt.Errorf("Unknown IP object: %v", ipc)
}
for _, route := range result.IP4.Routes {
if defaultNet.String() == route.Dst.String() {
if route.GW != nil && !route.GW.Equal(result.IP4.Gateway) {
return fmt.Errorf(
"isDefaultGateway ineffective because IPAM sets default route via %q",
route.GW,
)
for _, route := range result.Routes {
if defaultNet.String() == route.Dst.String() {
if route.GW != nil && !route.GW.Equal(ipc.Gateway) {
return fmt.Errorf(
"isDefaultGateway ineffective because IPAM sets default route via %q",
route.GW,
)
}
}
}
result.Routes = append(
result.Routes,
&types.Route{Dst: *defaultNet, GW: ipc.Gateway},
)
}
result.IP4.Routes = append(
result.IP4.Routes,
types.Route{Dst: *defaultNet, GW: result.IP4.Gateway},
)
// TODO: IPV6
}
if err := ipam.ConfigureIface(args.IfName, result); err != nil {
return err
}
if err := ip.SetHWAddrByIP(args.IfName, result.IP4.IP.IP, nil /* TODO IPv6 */); err != nil {
if err := ip.SetHWAddrByIP(args.IfName, result.IPs[0].Address.IP, nil /* TODO IPv6 */); err != nil {
return err
}
// Refetch the veth since its MAC address may changed
link, err := netlink.LinkByName(args.IfName)
if err != nil {
return fmt.Errorf("could not lookup %q: %v", args.IfName, err)
}
containerInterface.Mac = link.Attrs().HardwareAddr.String()
return nil
}); err != nil {
return err
}
if n.IsGW {
gwn := &net.IPNet{
IP: result.IP4.Gateway,
Mask: result.IP4.IP.Mask,
var firstV4Addr net.IP
for _, ipc := range result.IPs {
gwn := &net.IPNet{
IP: ipc.Gateway,
Mask: ipc.Address.Mask,
}
if ipc.Gateway.To4() != nil && firstV4Addr == nil {
firstV4Addr = ipc.Gateway
}
if err = ensureBridgeAddr(br, gwn, n.ForceAddress); err != nil {
return err
}
}
if err = ensureBridgeAddr(br, gwn, n.ForceAddress); err != nil {
return err
}
if err := ip.SetHWAddrByIP(n.BrName, gwn.IP, nil /* TODO IPv6 */); err != nil {
return err
if firstV4Addr != nil {
if err := ip.SetHWAddrByIP(n.BrName, firstV4Addr, nil /* TODO IPv6 */); err != nil {
return err
}
}
if err := ip.EnableIP4Forward(); err != nil {
@ -316,17 +352,28 @@ func cmdAdd(args *skel.CmdArgs) error {
if n.IPMasq {
chain := utils.FormatChainName(n.Name, args.ContainerID)
comment := utils.FormatComment(n.Name, args.ContainerID)
if err = ip.SetupIPMasq(ip.Network(&result.IP4.IP), chain, comment); err != nil {
return err
for _, ipc := range result.IPs {
if err = ip.SetupIPMasq(ip.Network(&ipc.Address), chain, comment); err != nil {
return err
}
}
}
// Refetch the bridge since its MAC address may change when the first
// veth is added or after its IP address is set
br, err = bridgeByName(n.BrName)
if err != nil {
return err
}
brInterface.Mac = br.Attrs().HardwareAddr.String()
result.DNS = n.DNS
return result.Print()
return types.PrintResult(result, cniVersion)
}
func cmdDel(args *skel.CmdArgs) error {
n, err := loadNetConf(args.StdinData)
n, _, err := loadNetConf(args.StdinData)
if err != nil {
return err
}
@ -361,5 +408,5 @@ func cmdDel(args *skel.CmdArgs) error {
}
func main() {
skel.PluginMain(cmdAdd, cmdDel, version.Legacy)
skel.PluginMain(cmdAdd, cmdDel, version.All)
}

204
plugins/main/bridge/bridge_test.go
View File

@ -17,12 +17,15 @@ package main
import (
"fmt"
"net"
"strings"
"syscall"
"github.com/containernetworking/cni/pkg/ns"
"github.com/containernetworking/cni/pkg/skel"
"github.com/containernetworking/cni/pkg/testutils"
"github.com/containernetworking/cni/pkg/types"
"github.com/containernetworking/cni/pkg/types/020"
"github.com/containernetworking/cni/pkg/types/current"
"github.com/containernetworking/cni/pkg/utils/hwaddr"
@ -51,7 +54,7 @@ var _ = Describe("bridge Operations", func() {
conf := &NetConf{
NetConf: types.NetConf{
CNIVersion: "0.2.0",
CNIVersion: "0.3.0",
Name: "testConfig",
Type: "bridge",
},
@ -64,7 +67,7 @@ var _ = Describe("bridge Operations", func() {
err := originalNS.Do(func(ns.NetNS) error {
defer GinkgoRecover()
bridge, err := setupBridge(conf)
bridge, _, err := setupBridge(conf)
Expect(err).NotTo(HaveOccurred())
Expect(bridge.Attrs().Name).To(Equal(IFNAME))
@ -96,7 +99,7 @@ var _ = Describe("bridge Operations", func() {
conf := &NetConf{
NetConf: types.NetConf{
CNIVersion: "0.2.0",
CNIVersion: "0.3.0",
Name: "testConfig",
Type: "bridge",
},
@ -105,7 +108,7 @@ var _ = Describe("bridge Operations", func() {
IPMasq: false,
}
bridge, err := setupBridge(conf)
bridge, _, err := setupBridge(conf)
Expect(err).NotTo(HaveOccurred())
Expect(bridge.Attrs().Name).To(Equal(IFNAME))
Expect(bridge.Attrs().Index).To(Equal(ifindex))
@ -128,7 +131,7 @@ var _ = Describe("bridge Operations", func() {
Expect(err).NotTo(HaveOccurred())
conf := fmt.Sprintf(`{
"cniVersion": "0.2.0",
"cniVersion": "0.3.0",
"name": "mynet",
"type": "bridge",
"bridge": "%s",
@ -151,18 +154,31 @@ var _ = Describe("bridge Operations", func() {
StdinData: []byte(conf),
}
var result *current.Result
err = originalNS.Do(func(ns.NetNS) error {
defer GinkgoRecover()
_, _, err := testutils.CmdAddWithResult(targetNs.Path(), IFNAME, []byte(conf), func() error {
r, raw, err := testutils.CmdAddWithResult(targetNs.Path(), IFNAME, []byte(conf), func() error {
return cmdAdd(args)
})
Expect(err).NotTo(HaveOccurred())
Expect(strings.Index(string(raw), "\"interfaces\":")).Should(BeNumerically(">", 0))
result, err = current.GetResult(r)
Expect(err).NotTo(HaveOccurred())
Expect(len(result.Interfaces)).To(Equal(3))
Expect(result.Interfaces[0].Name).To(Equal(BRNAME))
Expect(result.Interfaces[2].Name).To(Equal(IFNAME))
// Make sure bridge link exists
link, err := netlink.LinkByName(BRNAME)
link, err := netlink.LinkByName(result.Interfaces[0].Name)
Expect(err).NotTo(HaveOccurred())
Expect(link.Attrs().Name).To(Equal(BRNAME))
Expect(link).To(BeAssignableToTypeOf(&netlink.Bridge{}))
Expect(link.Attrs().HardwareAddr.String()).To(Equal(result.Interfaces[0].Mac))
hwAddr := fmt.Sprintf("%s", link.Attrs().HardwareAddr)
Expect(hwAddr).To(HavePrefix(hwaddr.PrivateMACPrefixString))
// Ensure bridge has gateway address
addrs, err := netlink.AddrList(link, syscall.AF_INET)
@ -183,23 +199,10 @@ var _ = Describe("bridge Operations", func() {
links, err := netlink.LinkList()
Expect(err).NotTo(HaveOccurred())
Expect(len(links)).To(Equal(3)) // Bridge, veth, and loopback
for _, l := range links {
switch {
case l.Attrs().Name == BRNAME:
{
_, isBridge := l.(*netlink.Bridge)
Expect(isBridge).To(Equal(true))
hwAddr := fmt.Sprintf("%s", l.Attrs().HardwareAddr)
Expect(hwAddr).To(HavePrefix(hwaddr.PrivateMACPrefixString))
}
case l.Attrs().Name != BRNAME && l.Attrs().Name != "lo":
{
_, isVeth := l.(*netlink.Veth)
Expect(isVeth).To(Equal(true))
}
}
}
link, err = netlink.LinkByName(result.Interfaces[1].Name)
Expect(err).NotTo(HaveOccurred())
Expect(link).To(BeAssignableToTypeOf(&netlink.Veth{}))
return nil
})
Expect(err).NotTo(HaveOccurred())
@ -211,6 +214,11 @@ var _ = Describe("bridge Operations", func() {
link, err := netlink.LinkByName(IFNAME)
Expect(err).NotTo(HaveOccurred())
Expect(link.Attrs().Name).To(Equal(IFNAME))
Expect(link).To(BeAssignableToTypeOf(&netlink.Veth{}))
addrs, err := netlink.AddrList(link, syscall.AF_INET)
Expect(err).NotTo(HaveOccurred())
Expect(len(addrs)).To(Equal(1))
hwAddr := fmt.Sprintf("%s", link.Attrs().HardwareAddr)
Expect(hwAddr).To(HavePrefix(hwaddr.PrivateMACPrefixString))
@ -243,7 +251,7 @@ var _ = Describe("bridge Operations", func() {
})
Expect(err).NotTo(HaveOccurred())
// Make sure macvlan link has been deleted
// Make sure the host veth has been deleted
err = targetNs.Do(func(ns.NetNS) error {
defer GinkgoRecover()
@ -253,6 +261,150 @@ var _ = Describe("bridge Operations", func() {
return nil
})
Expect(err).NotTo(HaveOccurred())
// Make sure the container veth has been deleted
err = originalNS.Do(func(ns.NetNS) error {
defer GinkgoRecover()
link, err := netlink.LinkByName(result.Interfaces[1].Name)
Expect(err).To(HaveOccurred())
Expect(link).To(BeNil())
return nil
})
})
It("configures and deconfigures a bridge and veth with default route with ADD/DEL for 0.1.0 config", func() {
const BRNAME = "cni0"
const IFNAME = "eth0"
gwaddr, subnet, err := net.ParseCIDR("10.1.2.1/24")
Expect(err).NotTo(HaveOccurred())
conf := fmt.Sprintf(`{
"cniVersion": "0.1.0",
"name": "mynet",
"type": "bridge",
"bridge": "%s",
"isDefaultGateway": true,
"ipMasq": false,
"ipam": {
"type": "host-local",
"subnet": "%s"
}
}`, BRNAME, subnet.String())
targetNs, err := ns.NewNS()
Expect(err).NotTo(HaveOccurred())
defer targetNs.Close()
args := &skel.CmdArgs{
ContainerID: "dummy",
Netns: targetNs.Path(),
IfName: IFNAME,
StdinData: []byte(conf),
}
var result *types020.Result
err = originalNS.Do(func(ns.NetNS) error {
defer GinkgoRecover()
r, raw, err := testutils.CmdAddWithResult(targetNs.Path(), IFNAME, []byte(conf), func() error {
return cmdAdd(args)
})
Expect(err).NotTo(HaveOccurred())
Expect(strings.Index(string(raw), "\"ip4\":")).Should(BeNumerically(">", 0))
// We expect a version 0.1.0 result
result, err = types020.GetResult(r)
Expect(err).NotTo(HaveOccurred())
// Make sure bridge link exists
link, err := netlink.LinkByName(BRNAME)
Expect(err).NotTo(HaveOccurred())
Expect(link.Attrs().Name).To(Equal(BRNAME))
Expect(link).To(BeAssignableToTypeOf(&netlink.Bridge{}))
hwAddr := fmt.Sprintf("%s", link.Attrs().HardwareAddr)
Expect(hwAddr).To(HavePrefix(hwaddr.PrivateMACPrefixString))
// Ensure bridge has gateway address
addrs, err := netlink.AddrList(link, syscall.AF_INET)
Expect(err).NotTo(HaveOccurred())
Expect(len(addrs)).To(BeNumerically(">", 0))
found := false
subnetPrefix, subnetBits := subnet.Mask.Size()
for _, a := range addrs {
aPrefix, aBits := a.IPNet.Mask.Size()
if a.IPNet.IP.Equal(gwaddr) && aPrefix == subnetPrefix && aBits == subnetBits {
found = true
break
}
}
Expect(found).To(Equal(true))
// Check for the veth link in the main namespace; can't
// check the for the specific link since version 0.1.0
// doesn't report interfaces
links, err := netlink.LinkList()
Expect(err).NotTo(HaveOccurred())
Expect(len(links)).To(Equal(3)) // Bridge, veth, and loopback
return nil
})
Expect(err).NotTo(HaveOccurred())
// Find the veth peer in the container namespace and the default route
err = targetNs.Do(func(ns.NetNS) error {
defer GinkgoRecover()
link, err := netlink.LinkByName(IFNAME)
Expect(err).NotTo(HaveOccurred())
Expect(link.Attrs().Name).To(Equal(IFNAME))
Expect(link).To(BeAssignableToTypeOf(&netlink.Veth{}))
addrs, err := netlink.AddrList(link, syscall.AF_INET)
Expect(err).NotTo(HaveOccurred())
Expect(len(addrs)).To(Equal(1))
hwAddr := fmt.Sprintf("%s", link.Attrs().HardwareAddr)
Expect(hwAddr).To(HavePrefix(hwaddr.PrivateMACPrefixString))
// Ensure the default route
routes, err := netlink.RouteList(link, 0)
Expect(err).NotTo(HaveOccurred())
var defaultRouteFound bool
for _, route := range routes {
defaultRouteFound = (route.Dst == nil && route.Src == nil && route.Gw.Equal(gwaddr))
if defaultRouteFound {
break
}
}
Expect(defaultRouteFound).To(Equal(true))
return nil
})
Expect(err).NotTo(HaveOccurred())
err = originalNS.Do(func(ns.NetNS) error {
defer GinkgoRecover()
err := testutils.CmdDelWithResult(targetNs.Path(), IFNAME, func() error {
return cmdDel(args)
})
Expect(err).NotTo(HaveOccurred())
return nil
})
Expect(err).NotTo(HaveOccurred())
// Make sure the container veth has been deleted; cannot check
// host veth as version 0.1.0 can't report its name
err = originalNS.Do(func(ns.NetNS) error {
defer GinkgoRecover()
link, err := netlink.LinkByName(IFNAME)
Expect(err).To(HaveOccurred())
Expect(link).To(BeNil())
return nil
})
})
It("ensure bridge address", func() {
@ -262,7 +414,7 @@ var _ = Describe("bridge Operations", func() {
conf := &NetConf{
NetConf: types.NetConf{
CNIVersion: "0.2.0",
CNIVersion: "0.3.0",
Name: "testConfig",
Type: "bridge",
},
@ -285,7 +437,7 @@ var _ = Describe("bridge Operations", func() {
err := originalNS.Do(func(ns.NetNS) error {
defer GinkgoRecover()
bridge, err := setupBridge(conf)
bridge, _, err := setupBridge(conf)
Expect(err).NotTo(HaveOccurred())
// Check if ForceAddress has default value
Expect(conf.ForceAddress).To(Equal(false))