From 3daee3214d8538eed40f1a011551965b211a4a75 Mon Sep 17 00:00:00 2001 From: Casey Callendrello Date: Mon, 12 Jun 2017 21:12:23 +0200 Subject: [PATCH] ptp: add ipv6 support * Wait for addresses to leave tentative state before setting routes * Enable forwarding correctly * Set up masquerading according to the active protocol --- pkg/ip/addr.go | 68 ++++++++++++++++++++++++++++++ pkg/ip/ipforward.go | 24 +++++++++++ pkg/ip/ipmasq.go | 38 ++++++++++++++--- pkg/ipam/ipam.go | 2 + pkg/testutils/ping.go | 55 ++++++++++++++++++++++++ plugins/main/ptp/ptp.go | 15 ++++--- plugins/main/ptp/ptp_test.go | 81 +++++++++++++++++++++++++++++------- 7 files changed, 258 insertions(+), 25 deletions(-) create mode 100644 pkg/ip/addr.go create mode 100644 pkg/testutils/ping.go diff --git a/pkg/ip/addr.go b/pkg/ip/addr.go new file mode 100644 index 00000000..b4db50b9 --- /dev/null +++ b/pkg/ip/addr.go @@ -0,0 +1,68 @@ +// Copyright 2017 CNI authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ip + +import ( + "fmt" + "syscall" + "time" + + "github.com/vishvananda/netlink" +) + +const SETTLE_INTERVAL = 50 * time.Millisecond + +// SettleAddresses waits for all addresses on a link to leave tentative state. +// This is particularly useful for ipv6, where all addresses need to do DAD. +// There is no easy way to wait for this as an event, so just loop until the +// addresses are no longer tentative. +// If any addresses are still tentative after timeout seconds, then error. +func SettleAddresses(ifName string, timeout int) error { + link, err := netlink.LinkByName(ifName) + if err != nil { + return fmt.Errorf("failed to retrieve link: %v", err) + } + + deadline := time.Now().Add(time.Duration(timeout) * time.Second) + for { + addrs, err := netlink.AddrList(link, netlink.FAMILY_ALL) + if err != nil { + return fmt.Errorf("could not list addresses: %v", err) + } + + if len(addrs) == 0 { + return nil + } + + ok := true + for _, addr := range addrs { + if addr.Flags&(syscall.IFA_F_TENTATIVE|syscall.IFA_F_DADFAILED) > 0 { + ok = false + break // Break out of the `range addrs`, not the `for` + } + } + + if ok { + return nil + } + if time.Now().After(deadline) { + return fmt.Errorf("link %s still has tentative addresses after %d seconds", + ifName, + timeout) + } + + time.Sleep(SETTLE_INTERVAL) + } +} diff --git a/pkg/ip/ipforward.go b/pkg/ip/ipforward.go index 77ee7463..abab3ecf 100644 --- a/pkg/ip/ipforward.go +++ b/pkg/ip/ipforward.go @@ -16,6 +16,8 @@ package ip import ( "io/ioutil" + + "github.com/containernetworking/cni/pkg/types/current" ) func EnableIP4Forward() error { @@ -26,6 +28,28 @@ func EnableIP6Forward() error { return echo1("/proc/sys/net/ipv6/conf/all/forwarding") } +// EnableForward will enable forwarding for all configured +// address families +func EnableForward(ips []*current.IPConfig) error { + v4 := false + v6 := false + + for _, ip := range ips { + if ip.Version == "4" && !v4 { + if err := EnableIP4Forward(); err != nil { + return err + } + v4 = true + } else if ip.Version == "6" && !v6 { + if err := EnableIP6Forward(); err != nil { + return err + } + v6 = true + } + } + return nil +} + func echo1(f string) error { return ioutil.WriteFile(f, []byte("1"), 0644) } diff --git a/pkg/ip/ipmasq.go b/pkg/ip/ipmasq.go index 8ee27971..7a549d13 100644 --- a/pkg/ip/ipmasq.go +++ b/pkg/ip/ipmasq.go @@ -24,23 +24,49 @@ import ( // SetupIPMasq installs iptables rules to masquerade traffic // coming from ipn and going outside of it func SetupIPMasq(ipn *net.IPNet, chain string, comment string) error { - ipt, err := iptables.New() + isV6 := ipn.IP.To4() == nil + + var ipt *iptables.IPTables + var err error + var multicastNet string + + if isV6 { + ipt, err = iptables.NewWithProtocol(iptables.ProtocolIPv6) + multicastNet = "ff00::/8" + } else { + ipt, err = iptables.NewWithProtocol(iptables.ProtocolIPv4) + multicastNet = "224.0.0.0/4" + } if err != nil { return fmt.Errorf("failed to locate iptables: %v", err) } - if err = ipt.NewChain("nat", chain); err != nil { - if err.(*iptables.Error).ExitStatus() != 1 { - // TODO(eyakubovich): assumes exit status 1 implies chain exists + // Create chain if doesn't exist + exists := false + chains, err := ipt.ListChains("nat") + if err != nil { + return fmt.Errorf("failed to list chains: %v", err) + } + for _, ch := range chains { + if ch == chain { + exists = true + break + } + } + if !exists { + if err = ipt.NewChain("nat", chain); err != nil { return err } } - if err = ipt.AppendUnique("nat", chain, "-d", ipn.String(), "-j", "ACCEPT", "-m", "comment", "--comment", comment); err != nil { + // Packets to this network should not be touched + if err := ipt.AppendUnique("nat", chain, "-d", ipn.String(), "-j", "ACCEPT", "-m", "comment", "--comment", comment); err != nil { return err } - if err = ipt.AppendUnique("nat", chain, "!", "-d", "224.0.0.0/4", "-j", "MASQUERADE", "-m", "comment", "--comment", comment); err != nil { + // Don't masquerade multicast - pods should be able to talk to other pods + // on the local network via multicast. + if err := ipt.AppendUnique("nat", chain, "!", "-d", multicastNet, "-j", "MASQUERADE", "-m", "comment", "--comment", comment); err != nil { return err } diff --git a/pkg/ipam/ipam.go b/pkg/ipam/ipam.go index 54f80c0e..ea7444a9 100644 --- a/pkg/ipam/ipam.go +++ b/pkg/ipam/ipam.go @@ -71,6 +71,8 @@ func ConfigureIface(ifName string, res *current.Result) error { } } + ip.SettleAddresses(ifName, 10) + for _, r := range res.Routes { routeIsV4 := r.Dst.IP.To4() != nil gw := r.GW diff --git a/pkg/testutils/ping.go b/pkg/testutils/ping.go new file mode 100644 index 00000000..5ee9db1c --- /dev/null +++ b/pkg/testutils/ping.go @@ -0,0 +1,55 @@ +// Copyright 2017 CNI authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package testutils + +import ( + "bytes" + "fmt" + "os/exec" + "strconv" + "syscall" +) + +// Ping shells out to the `ping` command. Returns nil if successful. +func Ping(saddr, daddr string, isV6 bool, timeoutSec int) error { + args := []string{ + "-c", "1", + "-W", strconv.Itoa(timeoutSec), + "-I", saddr, + daddr, + } + + bin := "ping" + if isV6 { + bin = "ping6" + } + + cmd := exec.Command(bin, args...) + var stderr bytes.Buffer + cmd.Stderr = &stderr + + if err := cmd.Run(); err != nil { + switch e := err.(type) { + case *exec.ExitError: + return fmt.Errorf("%v exit status %d: %s", + args, e.Sys().(syscall.WaitStatus).ExitStatus(), + stderr.String()) + default: + return err + } + } + + return nil +} diff --git a/plugins/main/ptp/ptp.go b/plugins/main/ptp/ptp.go index 42b26707..2ce2da2b 100644 --- a/plugins/main/ptp/ptp.go +++ b/plugins/main/ptp/ptp.go @@ -104,12 +104,17 @@ func setupContainerVeth(netns ns.NetNS, ifName string, mtu int, pr *current.Resu return fmt.Errorf("failed to delete route %v: %v", route, err) } + addrBits := 32 + if ipc.Version == "6" { + addrBits = 128 + } + for _, r := range []netlink.Route{ netlink.Route{ LinkIndex: contVeth.Index, Dst: &net.IPNet{ IP: ipc.Gateway, - Mask: net.CIDRMask(32, 32), + Mask: net.CIDRMask(addrBits, addrBits), }, Scope: netlink.SCOPE_LINK, Src: ipc.Address.IP, @@ -187,10 +192,6 @@ func cmdAdd(args *skel.CmdArgs) error { return fmt.Errorf("failed to load netconf: %v", err) } - if err := ip.EnableIP4Forward(); err != nil { - return fmt.Errorf("failed to enable forwarding: %v", err) - } - // run the IPAM plugin and get back the config to apply r, err := ipam.ExecAdd(conf.IPAM.Type, args.StdinData) if err != nil { @@ -206,6 +207,10 @@ func cmdAdd(args *skel.CmdArgs) error { return errors.New("IPAM plugin returned missing IP config") } + if err := ip.EnableForward(result.IPs); err != nil { + return fmt.Errorf("Could not enable IP forwarding: %v", err) + } + netns, err := ns.GetNS(args.Netns) if err != nil { return fmt.Errorf("failed to open netns %q: %v", args.Netns, err) diff --git a/plugins/main/ptp/ptp_test.go b/plugins/main/ptp/ptp_test.go index a562af7e..6132cd9a 100644 --- a/plugins/main/ptp/ptp_test.go +++ b/plugins/main/ptp/ptp_test.go @@ -15,7 +15,11 @@ package main import ( + "fmt" + "github.com/containernetworking/cni/pkg/skel" + "github.com/containernetworking/cni/pkg/types" + "github.com/containernetworking/cni/pkg/types/current" "github.com/containernetworking/plugins/pkg/ns" "github.com/containernetworking/plugins/pkg/testutils" @@ -39,21 +43,9 @@ var _ = Describe("ptp Operations", func() { Expect(originalNS.Close()).To(Succeed()) }) - It("configures and deconfigures a ptp link with ADD/DEL", func() { + doTest := func(conf string, numIPs int) { const IFNAME = "ptp0" - conf := `{ - "cniVersion": "0.3.1", - "name": "mynet", - "type": "ptp", - "ipMasq": true, - "mtu": 5000, - "ipam": { - "type": "host-local", - "subnet": "10.1.2.0/24" - } -}` - targetNs, err := ns.NewNS() Expect(err).NotTo(HaveOccurred()) defer targetNs.Close() @@ -65,11 +57,14 @@ var _ = Describe("ptp Operations", func() { StdinData: []byte(conf), } + var resI types.Result + var res *current.Result + // Execute the plugin with the ADD command, creating the veth endpoints err = originalNS.Do(func(ns.NetNS) error { defer GinkgoRecover() - _, _, err := testutils.CmdAddWithResult(targetNs.Path(), IFNAME, []byte(conf), func() error { + resI, _, err = testutils.CmdAddWithResult(targetNs.Path(), IFNAME, []byte(conf), func() error { return cmdAdd(args) }) Expect(err).NotTo(HaveOccurred()) @@ -77,17 +72,39 @@ var _ = Describe("ptp Operations", func() { }) Expect(err).NotTo(HaveOccurred()) + res, err = current.NewResultFromResult(resI) + Expect(err).NotTo(HaveOccurred()) + // Make sure ptp link exists in the target namespace + // Then, ping the gateway + seenIPs := 0 err = targetNs.Do(func(ns.NetNS) error { defer GinkgoRecover() link, err := netlink.LinkByName(IFNAME) Expect(err).NotTo(HaveOccurred()) Expect(link.Attrs().Name).To(Equal(IFNAME)) + + for _, ipc := range res.IPs { + if ipc.Interface != 1 { + continue + } + seenIPs += 1 + saddr := ipc.Address.IP.String() + daddr := ipc.Gateway.String() + fmt.Fprintln(GinkgoWriter, "ping", saddr, "->", daddr) + + if err := testutils.Ping(saddr, daddr, (ipc.Version == "6"), 30); err != nil { + return fmt.Errorf("ping %s -> %s failed: %s", saddr, daddr, err) + } + } + return nil }) Expect(err).NotTo(HaveOccurred()) + Expect(seenIPs).To(Equal(numIPs)) + // Call the plugins with the DEL command, deleting the veth endpoints err = originalNS.Do(func(ns.NetNS) error { defer GinkgoRecover() @@ -110,7 +127,43 @@ var _ = Describe("ptp Operations", func() { return nil }) Expect(err).NotTo(HaveOccurred()) + } + + It("configures and deconfigures a ptp link with ADD/DEL", func() { + conf := `{ + "cniVersion": "0.3.1", + "name": "mynet", + "type": "ptp", + "ipMasq": true, + "mtu": 5000, + "ipam": { + "type": "host-local", + "subnet": "10.1.2.0/24" + } +}` + + doTest(conf, 1) }) + + It("configures and deconfigures a dual-stack ptp link with ADD/DEL", func() { + conf := `{ + "cniVersion": "0.3.1", + "name": "mynet", + "type": "ptp", + "ipMasq": true, + "mtu": 5000, + "ipam": { + "type": "host-local", + "ranges": [ + { "subnet": "10.1.2.0/24"}, + { "subnet": "2001:db8:1::0/66"} + ] + } +}` + + doTest(conf, 2) + }) + It("deconfigures an unconfigured ptp link with DEL", func() { const IFNAME = "ptp0"