spec/plugins: return interface details and multiple IP addresses to runtime
Updates the spec and plugins to return an array of interfaces and IP details to the runtime including: - interface names and MAC addresses configured by the plugin - whether the interfaces are sandboxed (container/VM) or host (bridge, veth, etc) - multiple IP addresses configured by IPAM and which interface they have been assigned to Returning interface details is useful for runtimes, as well as allowing more flexible chaining of CNI plugins themselves. For example, some meta plugins may need to know the host-side interface to be able to apply firewall or traffic shaping rules to the container.
This commit is contained in:
Dan Williams
@ -53,14 +53,14 @@ func init() {
|
||||
runtime.LockOSThread()
|
||||
}
|
||||
|
||||
func loadNetConf(bytes []byte) (*NetConf, error) {
|
||||
func loadNetConf(bytes []byte) (*NetConf, string, error) {
|
||||
n := &NetConf{
|
||||
BrName: defaultBrName,
|
||||
}
|
||||
if err := json.Unmarshal(bytes, n); err != nil {
|
||||
return nil, fmt.Errorf("failed to load netconf: %v", err)
|
||||
return nil, "", fmt.Errorf("failed to load netconf: %v", err)
|
||||
}
|
||||
return n, nil
|
||||
return n, n.CNIVersion, nil
|
||||
}
|
||||
|
||||
func ensureBridgeAddr(br *netlink.Bridge, ipn *net.IPNet, forceAddress bool) error {
|
||||
@ -139,16 +139,16 @@ func ensureBridge(brName string, mtu int) (*netlink.Bridge, error) {
|
||||
},
|
||||
}
|
||||
|
||||
if err := netlink.LinkAdd(br); err != nil {
|
||||
if err != syscall.EEXIST {
|
||||
return nil, fmt.Errorf("could not add %q: %v", brName, err)
|
||||
}
|
||||
err := netlink.LinkAdd(br)
|
||||
if err != nil && err != syscall.EEXIST {
|
||||
return nil, fmt.Errorf("could not add %q: %v", brName, err)
|
||||
}
|
||||
|
||||
// it's ok if the device already exists as long as config is similar
|
||||
br, err = bridgeByName(brName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Re-fetch link to read all attributes and if it already existed,
|
||||
// ensure it's really a bridge with similar configuration
|
||||
br, err = bridgeByName(brName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err := netlink.LinkSetUp(br); err != nil {
|
||||
@ -158,40 +158,44 @@ func ensureBridge(brName string, mtu int) (*netlink.Bridge, error) {
|
||||
return br, nil
|
||||
}
|
||||
|
||||
func setupVeth(netns ns.NetNS, br *netlink.Bridge, ifName string, mtu int, hairpinMode bool) error {
|
||||
var hostVethName string
|
||||
func setupVeth(netns ns.NetNS, br *netlink.Bridge, ifName string, mtu int, hairpinMode bool) (*current.Interface, *current.Interface, error) {
|
||||
contIface := ¤t.Interface{}
|
||||
hostIface := ¤t.Interface{}
|
||||
|
||||
err := netns.Do(func(hostNS ns.NetNS) error {
|
||||
// create the veth pair in the container and move host end into host netns
|
||||
hostVeth, _, err := ip.SetupVeth(ifName, mtu, hostNS)
|
||||
hostVeth, containerVeth, err := ip.SetupVeth(ifName, mtu, hostNS)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
hostVethName = hostVeth.Attrs().Name
|
||||
contIface.Name = containerVeth.Attrs().Name
|
||||
contIface.Mac = containerVeth.Attrs().HardwareAddr.String()
|
||||
contIface.Sandbox = netns.Path()
|
||||
hostIface.Name = hostVeth.Attrs().Name
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
// need to lookup hostVeth again as its index has changed during ns move
|
||||
hostVeth, err := netlink.LinkByName(hostVethName)
|
||||
hostVeth, err := netlink.LinkByName(hostIface.Name)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to lookup %q: %v", hostVethName, err)
|
||||
return nil, nil, fmt.Errorf("failed to lookup %q: %v", hostIface.Name, err)
|
||||
}
|
||||
hostIface.Mac = hostVeth.Attrs().HardwareAddr.String()
|
||||
|
||||
// connect host veth end to the bridge
|
||||
if err = netlink.LinkSetMaster(hostVeth, br); err != nil {
|
||||
return fmt.Errorf("failed to connect %q to bridge %v: %v", hostVethName, br.Attrs().Name, err)
|
||||
if err := netlink.LinkSetMaster(hostVeth, br); err != nil {
|
||||
return nil, nil, fmt.Errorf("failed to connect %q to bridge %v: %v", hostVeth.Attrs().Name, br.Attrs().Name, err)
|
||||
}
|
||||
|
||||
// set hairpin mode
|
||||
if err = netlink.LinkSetHairpin(hostVeth, hairpinMode); err != nil {
|
||||
return fmt.Errorf("failed to setup hairpin mode for %v: %v", hostVethName, err)
|
||||
return nil, nil, fmt.Errorf("failed to setup hairpin mode for %v: %v", hostVeth.Attrs().Name, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
return hostIface, contIface, nil
|
||||
}
|
||||
|
||||
func calcGatewayIP(ipn *net.IPNet) net.IP {
|
||||
@ -199,18 +203,21 @@ func calcGatewayIP(ipn *net.IPNet) net.IP {
|
||||
return ip.NextIP(nid)
|
||||
}
|
||||
|
||||
func setupBridge(n *NetConf) (*netlink.Bridge, error) {
|
||||
func setupBridge(n *NetConf) (*netlink.Bridge, *current.Interface, error) {
|
||||
// create bridge if necessary
|
||||
br, err := ensureBridge(n.BrName, n.MTU)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create bridge %q: %v", n.BrName, err)
|
||||
return nil, nil, fmt.Errorf("failed to create bridge %q: %v", n.BrName, err)
|
||||
}
|
||||
|
||||
return br, nil
|
||||
return br, ¤t.Interface{
|
||||
Name: br.Attrs().Name,
|
||||
Mac: br.Attrs().HardwareAddr.String(),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func cmdAdd(args *skel.CmdArgs) error {
|
||||
n, err := loadNetConf(args.StdinData)
|
||||
n, cniVersion, err := loadNetConf(args.StdinData)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@ -219,7 +226,7 @@ func cmdAdd(args *skel.CmdArgs) error {
|
||||
n.IsGW = true
|
||||
}
|
||||
|
||||
br, err := setupBridge(n)
|
||||
br, brInterface, err := setupBridge(n)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@ -230,7 +237,8 @@ func cmdAdd(args *skel.CmdArgs) error {
|
||||
}
|
||||
defer netns.Close()
|
||||
|
||||
if err = setupVeth(netns, br, args.IfName, n.MTU, n.HairpinMode); err != nil {
|
||||
hostInterface, containerInterface, err := setupVeth(netns, br, args.IfName, n.MTU, n.HairpinMode)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@ -240,72 +248,100 @@ func cmdAdd(args *skel.CmdArgs) error {
|
||||
return err
|
||||
}
|
||||
|
||||
result, err := current.GetResult(r)
|
||||
// Convert whatever the IPAM result was into the current Result type
|
||||
result, err := current.NewResultFromResult(r)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// TODO: make this optional when IPv6 is supported
|
||||
if result.IP4 == nil {
|
||||
return errors.New("IPAM plugin returned missing IPv4 config")
|
||||
if len(result.IPs) == 0 {
|
||||
return errors.New("IPAM plugin returned missing IP config")
|
||||
}
|
||||
|
||||
if result.IP4.Gateway == nil && n.IsGW {
|
||||
result.IP4.Gateway = calcGatewayIP(&result.IP4.IP)
|
||||
result.Interfaces = []*current.Interface{brInterface, hostInterface, containerInterface}
|
||||
|
||||
for _, ipc := range result.IPs {
|
||||
// All IPs currently refer to the container interface
|
||||
ipc.Interface = 2
|
||||
if ipc.Gateway == nil && n.IsGW {
|
||||
ipc.Gateway = calcGatewayIP(&ipc.Address)
|
||||
}
|
||||
}
|
||||
|
||||
if err := netns.Do(func(_ ns.NetNS) error {
|
||||
// set the default gateway if requested
|
||||
if n.IsDefaultGW {
|
||||
_, defaultNet, err := net.ParseCIDR("0.0.0.0/0")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, ipc := range result.IPs {
|
||||
defaultNet := &net.IPNet{}
|
||||
switch {
|
||||
case ipc.Address.IP.To4() != nil:
|
||||
defaultNet.IP = net.IPv4zero
|
||||
defaultNet.Mask = net.IPMask(net.IPv4zero)
|
||||
case len(ipc.Address.IP) == net.IPv6len && ipc.Address.IP.To4() == nil:
|
||||
defaultNet.IP = net.IPv6zero
|
||||
defaultNet.Mask = net.IPMask(net.IPv6zero)
|
||||
default:
|
||||
return fmt.Errorf("Unknown IP object: %v", ipc)
|
||||
}
|
||||
|
||||
for _, route := range result.IP4.Routes {
|
||||
if defaultNet.String() == route.Dst.String() {
|
||||
if route.GW != nil && !route.GW.Equal(result.IP4.Gateway) {
|
||||
return fmt.Errorf(
|
||||
"isDefaultGateway ineffective because IPAM sets default route via %q",
|
||||
route.GW,
|
||||
)
|
||||
for _, route := range result.Routes {
|
||||
if defaultNet.String() == route.Dst.String() {
|
||||
if route.GW != nil && !route.GW.Equal(ipc.Gateway) {
|
||||
return fmt.Errorf(
|
||||
"isDefaultGateway ineffective because IPAM sets default route via %q",
|
||||
route.GW,
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
result.Routes = append(
|
||||
result.Routes,
|
||||
&types.Route{Dst: *defaultNet, GW: ipc.Gateway},
|
||||
)
|
||||
}
|
||||
|
||||
result.IP4.Routes = append(
|
||||
result.IP4.Routes,
|
||||
types.Route{Dst: *defaultNet, GW: result.IP4.Gateway},
|
||||
)
|
||||
|
||||
// TODO: IPV6
|
||||
}
|
||||
|
||||
if err := ipam.ConfigureIface(args.IfName, result); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := ip.SetHWAddrByIP(args.IfName, result.IP4.IP.IP, nil /* TODO IPv6 */); err != nil {
|
||||
if err := ip.SetHWAddrByIP(args.IfName, result.IPs[0].Address.IP, nil /* TODO IPv6 */); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Refetch the veth since its MAC address may changed
|
||||
link, err := netlink.LinkByName(args.IfName)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not lookup %q: %v", args.IfName, err)
|
||||
}
|
||||
containerInterface.Mac = link.Attrs().HardwareAddr.String()
|
||||
|
||||
return nil
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if n.IsGW {
|
||||
gwn := &net.IPNet{
|
||||
IP: result.IP4.Gateway,
|
||||
Mask: result.IP4.IP.Mask,
|
||||
var firstV4Addr net.IP
|
||||
for _, ipc := range result.IPs {
|
||||
gwn := &net.IPNet{
|
||||
IP: ipc.Gateway,
|
||||
Mask: ipc.Address.Mask,
|
||||
}
|
||||
if ipc.Gateway.To4() != nil && firstV4Addr == nil {
|
||||
firstV4Addr = ipc.Gateway
|
||||
}
|
||||
|
||||
if err = ensureBridgeAddr(br, gwn, n.ForceAddress); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if err = ensureBridgeAddr(br, gwn, n.ForceAddress); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := ip.SetHWAddrByIP(n.BrName, gwn.IP, nil /* TODO IPv6 */); err != nil {
|
||||
return err
|
||||
if firstV4Addr != nil {
|
||||
if err := ip.SetHWAddrByIP(n.BrName, firstV4Addr, nil /* TODO IPv6 */); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if err := ip.EnableIP4Forward(); err != nil {
|
||||
@ -316,17 +352,28 @@ func cmdAdd(args *skel.CmdArgs) error {
|
||||
if n.IPMasq {
|
||||
chain := utils.FormatChainName(n.Name, args.ContainerID)
|
||||
comment := utils.FormatComment(n.Name, args.ContainerID)
|
||||
if err = ip.SetupIPMasq(ip.Network(&result.IP4.IP), chain, comment); err != nil {
|
||||
return err
|
||||
for _, ipc := range result.IPs {
|
||||
if err = ip.SetupIPMasq(ip.Network(&ipc.Address), chain, comment); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Refetch the bridge since its MAC address may change when the first
|
||||
// veth is added or after its IP address is set
|
||||
br, err = bridgeByName(n.BrName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
brInterface.Mac = br.Attrs().HardwareAddr.String()
|
||||
|
||||
result.DNS = n.DNS
|
||||
return result.Print()
|
||||
|
||||
return types.PrintResult(result, cniVersion)
|
||||
}
|
||||
|
||||
func cmdDel(args *skel.CmdArgs) error {
|
||||
n, err := loadNetConf(args.StdinData)
|
||||
n, _, err := loadNetConf(args.StdinData)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@ -361,5 +408,5 @@ func cmdDel(args *skel.CmdArgs) error {
|
||||
}
|
||||
|
||||
func main() {
|
||||
skel.PluginMain(cmdAdd, cmdDel, version.Legacy)
|
||||
skel.PluginMain(cmdAdd, cmdDel, version.All)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user