diff --git a/README.md b/README.md index 5aa1dd55..09e25603 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ Some CNI network plugins, maintained by the containernetworking team. For more i ### Meta: other plugins * `flannel`: generates an interface corresponding to a flannel config file * `tuning`: Tweaks sysctl parameters of an existing interface - +* `portmap`: An iptables-based portmapping plugin. Maps ports from the host's address space to the container. ### Sample The sample plugin provides an example for building your own plugin. diff --git a/plugins/meta/portmap/README.md b/plugins/meta/portmap/README.md new file mode 100644 index 00000000..fc6b86c9 --- /dev/null +++ b/plugins/meta/portmap/README.md @@ -0,0 +1,117 @@ +## Port-mapping plugin + +This plugin will forward traffic from one or more ports on the host to the +container. It expects to be run as a chained plugin. + +## Usage +You should use this plugin as part of a network configuration list. It accepts +the following configuration options: + +* `snat` - boolean, default true. If true or omitted, set up the SNAT chains +* `conditionsV4`, `conditionsV6` - array of strings. A list of arbitrary `iptables` +matches to add to the per-container rule. This may be useful if you wish to +exclude specific IPs from port-mapping + +The plugin expects to receive the actual list of port mappings via the +`portMappings` [capability argument](https://github.com/containernetworking/cni/blob/master/CONVENTIONS.md) + +So a sample standalone config list (with the file extension .conflist) might +look like: + +```json +{ + "cniVersion": "0.3.1", + "name": "mynet", + "plugins": [ + { + "type": "ptp", + "ipMasq": true, + "ipam": { + "type": "host-local", + "subnet": "172.16.30.0/24", + "routes": [ + { + "dst": "0.0.0.0/0" + } + ] + } + }, + { + "type": "portmap", + "capabilities": {"portMappings": true}, + "snat": false, + "conditionsV4": ["!", "-d", "192.0.2.0/24"], + "conditionsV6": ["!", "-d", "fc00::/7"] + } + ] +} +``` + + + +## Rule structure +The plugin sets up two sequences of chains and rules - one "primary" DNAT +sequence to rewrite the destination, and one additional SNAT sequence that +rewrites the source address for packets from localhost. The sequence is somewhat +complex to minimize the number of rules non-forwarded packets must traverse. + + +### DNAT +The DNAT rule rewrites the destination port and address of new connections. +There is a top-level chain, `CNI-HOSTPORT-DNAT` which is always created and +never deleted. Each plugin execution creates an additional chain for ease +of cleanup. So, if a single container exists on IP 172.16.30.2 with ports +8080 and 8043 on the host forwarded to ports 80 and 443 in the container, the +rules look like this: + +`PREROUTING`, `OUTPUT` chains: +- `--dst-type LOCAL -j CNI-HOSTPORT-DNAT` + +`CNI-HOSTPORT-DNAT` chain: +- `${ConditionsV4/6} -j CNI-DN-xxxxxx` (where xxxxxx is a function of the ContainerID and network name) + +`CNI-DN-xxxxxx` chain: +- `-p tcp --dport 8080 -j DNAT --to-destination 172.16.30.2:80` +- `-p tcp --dport 8043 -j DNAT --to-destination 172.16.30.2:443` + +New connections to the host will have to traverse every rule, so large numbers +of port forwards may have a performance impact. This won't affect established +connections, just the first packet. + +### SNAT +The SNAT rule enables port-forwarding from the localhost IP on the host. +This rule rewrites (masquerades) the source address for connections from +localhost. If this rule did not exist, a connection to `localhost:80` would +still have a source IP of 127.0.0.1 when received by the container, so no +packets would respond. Again, it is a sequence of 3 chains. Because SNAT has to +occur in the `POSTROUTING` chain, the packet has already been through the DNAT +chain. + +`POSTROUTING`: +- `-s 127.0.0.1 ! -d 127.0.0.1 -j CNI-HOSTPORT-SNAT` + +`CNI-HOSTPORT-SNAT`: +- `-j CNI-SN-xxxxx` + +`CNI-SN-xxxxx`: +- `-p tcp -s 127.0.0.1 -d 172.16.30.2 --dport 80 -j MASQUERADE` +- `-p tcp -s 127.0.0.1 -d 172.16.30.2 --dport 443 -j MASQUERADE` + +Only new connections from the host, where the source address is 127.0.0.1 but +not the destination will traverse this chain. It is unlikely that any packets +will reach these rules without being SNATted, so the cost should be minimal. + +Because MASQUERADE happens in POSTROUTING, it means that packets with source ip +127.0.0.1 need to pass a routing boundary. By default, that is not allowed +in Linux. So, need to enable the sysctl `net.ipv4.conf.IFNAME.route_localnet`, +where IFNAME is the name of the host-side interface that routes traffic to the +container. + +There is no equivalent to `route_localnet` for ipv6, so SNAT does not work +for ipv6. If you need port forwarding from localhost, your container must have +an ipv4 address. + + +## Known issues +- ipsets could improve efficiency +- SNAT does not work with ipv6. diff --git a/plugins/meta/portmap/chain.go b/plugins/meta/portmap/chain.go new file mode 100644 index 00000000..f8a53a45 --- /dev/null +++ b/plugins/meta/portmap/chain.go @@ -0,0 +1,127 @@ +// Copyright 2017 CNI authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "fmt" + "strings" + + "github.com/coreos/go-iptables/iptables" + shellwords "github.com/mattn/go-shellwords" +) + +type chain struct { + table string + name string + entryRule []string // the rule that enters this chain + entryChains []string // the chains to add the entry rule +} + +// setup idempotently creates the chain. It will not error if the chain exists. +func (c *chain) setup(ipt *iptables.IPTables, rules [][]string) error { + // create the chain + exists, err := chainExists(ipt, c.table, c.name) + if err != nil { + return err + } + if !exists { + if err := ipt.NewChain(c.table, c.name); err != nil { + return err + } + } + + // Add the rules to the chain + for i := len(rules) - 1; i >= 0; i-- { + if err := prependUnique(ipt, c.table, c.name, rules[i]); err != nil { + return err + } + } + + // Add the entry rules + entryRule := append(c.entryRule, "-j", c.name) + for _, entryChain := range c.entryChains { + if err := prependUnique(ipt, c.table, entryChain, entryRule); err != nil { + return err + } + } + + return nil +} + +// teardown idempotently deletes a chain. It will not error if the chain doesn't exist. +// It will first delete all references to this chain in the entryChains. +func (c *chain) teardown(ipt *iptables.IPTables) error { + // flush the chain + // This will succeed *and create the chain* if it does not exist. + // If the chain doesn't exist, the next checks will fail. + if err := ipt.ClearChain(c.table, c.name); err != nil { + return err + } + + for _, entryChain := range c.entryChains { + entryChainRules, err := ipt.List(c.table, entryChain) + if err != nil { + // Swallow error here - probably the chain doesn't exist. + // If we miss something the deletion will fail + continue + } + + for _, entryChainRule := range entryChainRules[1:] { + if strings.HasSuffix(entryChainRule, "-j "+c.name) { + chainParts, err := shellwords.Parse(entryChainRule) + if err != nil { + return fmt.Errorf("error parsing iptables rule: %s: %v", entryChainRule, err) + } + chainParts = chainParts[2:] // List results always include an -A CHAINNAME + + if err := ipt.Delete(c.table, entryChain, chainParts...); err != nil { + return fmt.Errorf("Failed to delete referring rule %s %s: %v", c.table, entryChainRule, err) + } + } + } + } + + if err := ipt.DeleteChain(c.table, c.name); err != nil { + return err + } + return nil +} + +// prependUnique will prepend a rule to a chain, if it does not already exist +func prependUnique(ipt *iptables.IPTables, table, chain string, rule []string) error { + exists, err := ipt.Exists(table, chain, rule...) + if err != nil { + return err + } + if exists { + return nil + } + + return ipt.Insert(table, chain, 1, rule...) +} + +func chainExists(ipt *iptables.IPTables, tableName, chainName string) (bool, error) { + chains, err := ipt.ListChains(tableName) + if err != nil { + return false, err + } + + for _, ch := range chains { + if ch == chainName { + return true, nil + } + } + return false, nil +} diff --git a/plugins/meta/portmap/chain_test.go b/plugins/meta/portmap/chain_test.go new file mode 100644 index 00000000..5cc4cf64 --- /dev/null +++ b/plugins/meta/portmap/chain_test.go @@ -0,0 +1,203 @@ +// Copyright 2017 CNI authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "fmt" + "math/rand" + "runtime" + + "github.com/containernetworking/plugins/pkg/ns" + "github.com/coreos/go-iptables/iptables" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +const TABLE = "filter" // We'll monkey around here + +// TODO: run these tests in a new namespace +var _ = Describe("chain tests", func() { + var testChain chain + var ipt *iptables.IPTables + var cleanup func() + + BeforeEach(func() { + + // Save a reference to the original namespace, + // Add a new NS + currNs, err := ns.GetCurrentNS() + Expect(err).NotTo(HaveOccurred()) + + testNs, err := ns.NewNS() + Expect(err).NotTo(HaveOccurred()) + + tlChainName := fmt.Sprintf("cni-test-%d", rand.Intn(10000000)) + chainName := fmt.Sprintf("cni-test-%d", rand.Intn(10000000)) + + testChain = chain{ + table: TABLE, + name: chainName, + entryRule: []string{"-d", "203.0.113.1"}, + entryChains: []string{tlChainName}, + } + + ipt, err = iptables.NewWithProtocol(iptables.ProtocolIPv4) + Expect(err).NotTo(HaveOccurred()) + + runtime.LockOSThread() + err = testNs.Set() + Expect(err).NotTo(HaveOccurred()) + + err = ipt.ClearChain(TABLE, tlChainName) // This will create the chain + if err != nil { + currNs.Set() + Expect(err).NotTo(HaveOccurred()) + } + + cleanup = func() { + if ipt == nil { + return + } + ipt.ClearChain(TABLE, testChain.name) + ipt.ClearChain(TABLE, tlChainName) + ipt.DeleteChain(TABLE, testChain.name) + ipt.DeleteChain(TABLE, tlChainName) + currNs.Set() + } + + }) + + It("creates and destroys a chain", func() { + defer cleanup() + + tlChainName := testChain.entryChains[0] + + // add an extra rule to the test chain to make sure it's not touched + err := ipt.Append(TABLE, tlChainName, "-m", "comment", "--comment", + "canary value", "-j", "ACCEPT") + Expect(err).NotTo(HaveOccurred()) + + // Create the chain + chainRules := [][]string{ + {"-m", "comment", "--comment", "test 1", "-j", "RETURN"}, + {"-m", "comment", "--comment", "test 2", "-j", "RETURN"}, + } + err = testChain.setup(ipt, chainRules) + Expect(err).NotTo(HaveOccurred()) + + // Verify the chain exists + ok := false + chains, err := ipt.ListChains(TABLE) + Expect(err).NotTo(HaveOccurred()) + for _, chain := range chains { + if chain == testChain.name { + ok = true + break + } + } + if !ok { + Fail("Could not find created chain") + } + + // Check that the entry rule was created + haveRules, err := ipt.List(TABLE, tlChainName) + Expect(err).NotTo(HaveOccurred()) + Expect(haveRules).To(Equal([]string{ + "-N " + tlChainName, + "-A " + tlChainName + " -d 203.0.113.1/32 -j " + testChain.name, + "-A " + tlChainName + ` -m comment --comment "canary value" -j ACCEPT`, + })) + + // Check that the chain and rule was created + haveRules, err = ipt.List(TABLE, testChain.name) + Expect(err).NotTo(HaveOccurred()) + Expect(haveRules).To(Equal([]string{ + "-N " + testChain.name, + "-A " + testChain.name + ` -m comment --comment "test 1" -j RETURN`, + "-A " + testChain.name + ` -m comment --comment "test 2" -j RETURN`, + })) + + err = testChain.teardown(ipt) + Expect(err).NotTo(HaveOccurred()) + + tlRules, err := ipt.List(TABLE, tlChainName) + Expect(err).NotTo(HaveOccurred()) + Expect(tlRules).To(Equal([]string{ + "-N " + tlChainName, + "-A " + tlChainName + ` -m comment --comment "canary value" -j ACCEPT`, + })) + + chains, err = ipt.ListChains(TABLE) + Expect(err).NotTo(HaveOccurred()) + for _, chain := range chains { + if chain == testChain.name { + Fail("chain was not deleted") + } + } + }) + + It("creates chains idempotently", func() { + defer cleanup() + + // Create the chain + chainRules := [][]string{ + {"-m", "comment", "--comment", "test", "-j", "RETURN"}, + } + err := testChain.setup(ipt, chainRules) + Expect(err).NotTo(HaveOccurred()) + + // Create it again! + err = testChain.setup(ipt, chainRules) + Expect(err).NotTo(HaveOccurred()) + + // Make sure there are only two rules + // (the first rule is an -N because go-iptables + rules, err := ipt.List(TABLE, testChain.name) + Expect(err).NotTo(HaveOccurred()) + + Expect(len(rules)).To(Equal(2)) + + }) + + It("deletes chains idempotently", func() { + defer cleanup() + + // Create the chain + chainRules := [][]string{ + {"-m", "comment", "--comment", "test", "-j", "RETURN"}, + } + err := testChain.setup(ipt, chainRules) + Expect(err).NotTo(HaveOccurred()) + + err = testChain.teardown(ipt) + Expect(err).NotTo(HaveOccurred()) + + chains, err := ipt.ListChains(TABLE) + for _, chain := range chains { + if chain == testChain.name { + Fail("Chain was not deleted") + } + } + + err = testChain.teardown(ipt) + Expect(err).NotTo(HaveOccurred()) + chains, err = ipt.ListChains(TABLE) + for _, chain := range chains { + if chain == testChain.name { + Fail("Chain was not deleted") + } + } + }) +}) diff --git a/plugins/meta/portmap/main.go b/plugins/meta/portmap/main.go new file mode 100644 index 00000000..c0c34ae5 --- /dev/null +++ b/plugins/meta/portmap/main.go @@ -0,0 +1,159 @@ +// Copyright 2017 CNI authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// This is a post-setup plugin that establishes port forwarding - using iptables, +// from the host's network interface(s) to a pod's network interface. +// +// It is intended to be used as a chained CNI plugin, and determines the container +// IP from the previous result. If the result includes an IPv6 address, it will +// also be configured. (IPTables will not forward cross-family). +// +// This has one notable limitation: it does not perform any kind of reservation +// of the actual host port. If there is a service on the host, it will have all +// its traffic captured by the container. If another container also claims a given +// port, it will caputure the traffic - it is last-write-wins. +package main + +import ( + "encoding/json" + "fmt" + + "github.com/containernetworking/cni/pkg/skel" + "github.com/containernetworking/cni/pkg/types" + "github.com/containernetworking/cni/pkg/types/current" + "github.com/containernetworking/cni/pkg/version" +) + +// PortMapEntry corresponds to a single entry in the port_mappings argument, +// see CONVENTIONS.md +type PortMapEntry struct { + HostPort int `json:"hostPort"` + ContainerPort int `json:"containerPort"` + Protocol string `json:"protocol"` + HostIP string `json:"hostIP,omitempty"` +} + +type PortMapConf struct { + types.NetConf + SNAT *bool `json:"snat,omitempty"` + ConditionsV4 *[]string `json:"conditionsV4"` + ConditionsV6 *[]string `json:"conditionsV6"` + RuntimeConfig struct { + PortMaps []PortMapEntry `json:"portMappings,omitempty"` + } `json:"runtimeConfig,omitempty"` + RawPrevResult map[string]interface{} `json:"prevResult,omitempty"` + PrevResult *current.Result `json:"-"` + ContainerID string +} + +func cmdAdd(args *skel.CmdArgs) error { + netConf, err := parseConfig(args.StdinData) + if err != nil { + return fmt.Errorf("failed to parse config: %v", err) + } + + if netConf.PrevResult == nil { + return fmt.Errorf("must be called as chained plugin") + } + + if len(netConf.RuntimeConfig.PortMaps) == 0 { + return types.PrintResult(netConf.PrevResult, netConf.CNIVersion) + } + + netConf.ContainerID = args.ContainerID + + // Loop through IPs, setting up forwarding to the first container IP + // per family + hasV4 := false + hasV6 := false + for _, ip := range netConf.PrevResult.IPs { + if ip.Version == "6" && hasV6 { + continue + } else if ip.Version == "4" && hasV4 { + continue + } + + // Skip known non-sandbox interfaces + intIdx := ip.Interface + if intIdx >= 0 && intIdx < len(netConf.PrevResult.Interfaces) && netConf.PrevResult.Interfaces[intIdx].Name != args.IfName { + continue + } + + if err := forwardPorts(netConf, ip.Address.IP); err != nil { + return err + } + + if ip.Version == "6" { + hasV6 = true + } else { + hasV4 = true + } + } + + // Pass through the previous result + return types.PrintResult(netConf.PrevResult, netConf.CNIVersion) +} + +func cmdDel(args *skel.CmdArgs) error { + netConf, err := parseConfig(args.StdinData) + if err != nil { + return fmt.Errorf("failed to parse config: %v", err) + } + + netConf.ContainerID = args.ContainerID + + // We don't need to parse out whether or not we're using v6 or snat, + // deletion is idempotent + if err := unforwardPorts(netConf); err != nil { + return err + } + return nil +} + +func main() { + skel.PluginMain(cmdAdd, cmdDel, version.PluginSupports("", "0.1.0", "0.2.0", "0.3.0", version.Current())) +} + +// parseConfig parses the supplied configuration (and prevResult) from stdin. +func parseConfig(stdin []byte) (*PortMapConf, error) { + conf := PortMapConf{} + + if err := json.Unmarshal(stdin, &conf); err != nil { + return nil, fmt.Errorf("failed to parse network configuration: %v", err) + } + + // Parse previous result. + if conf.RawPrevResult != nil { + resultBytes, err := json.Marshal(conf.RawPrevResult) + if err != nil { + return nil, fmt.Errorf("could not serialize prevResult: %v", err) + } + res, err := version.NewResult(conf.CNIVersion, resultBytes) + if err != nil { + return nil, fmt.Errorf("could not parse prevResult: %v", err) + } + conf.RawPrevResult = nil + conf.PrevResult, err = current.NewResultFromResult(res) + if err != nil { + return nil, fmt.Errorf("could not convert result to current version: %v", err) + } + } + + if conf.SNAT == nil { + tvar := true + conf.SNAT = &tvar + } + + return &conf, nil +} diff --git a/plugins/meta/portmap/portmap.go b/plugins/meta/portmap/portmap.go new file mode 100644 index 00000000..133dfef2 --- /dev/null +++ b/plugins/meta/portmap/portmap.go @@ -0,0 +1,294 @@ +// Copyright 2017 CNI authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "fmt" + "net" + "strconv" + + "github.com/containernetworking/plugins/pkg/utils/sysctl" + "github.com/coreos/go-iptables/iptables" +) + +// This creates the chains to be added to iptables. The basic structure is +// a bit complex for efficiencies sake. We create 2 chains: a summary chain +// that is shared between invocations, and an invocation (container)-specific +// chain. This minimizes the number of operations on the top level, but allows +// for easy cleanup. +// +// We also create DNAT chains to rewrite destinations, and SNAT chains so that +// connections to localhost work. +// +// The basic setup (all operations are on the nat table) is: +// +// DNAT case (rewrite destination IP and port): +// PREROUTING, OUTPUT: --dst-type local -j CNI-HOSTPORT_DNAT +// CNI-HOSTPORT-DNAT: -j CNI-DN-abcd123 +// CNI-DN-abcd123: -p tcp --dport 8080 -j DNAT --to-destination 192.0.2.33:80 +// CNI-DN-abcd123: -p tcp --dport 8081 -j DNAT ... +// +// SNAT case (rewrite source IP from localhost after dnat): +// POSTROUTING: -s 127.0.0.1 ! -d 127.0.0.1 -j CNI-HOSTPORT-SNAT +// CNI-HOSTPORT-SNAT: -j CNI-SN-abcd123 +// CNI-SN-abcd123: -p tcp -s 127.0.0.1 -d 192.0.2.33 --dport 80 -j MASQUERADE +// CNI-SN-abcd123: -p tcp -s 127.0.0.1 -d 192.0.2.33 --dport 90 -j MASQUERADE + +// The names of the top-level summary chains. +// These should never be changed, or else upgrading will require manual +// intervention. +const TopLevelDNATChainName = "CNI-HOSTPORT-DNAT" +const TopLevelSNATChainName = "CNI-HOSTPORT-SNAT" + +// forwardPorts establishes port forwarding to a given container IP. +// containerIP can be either v4 or v6. +func forwardPorts(config *PortMapConf, containerIP net.IP) error { + isV6 := (containerIP.To4() == nil) + + var ipt *iptables.IPTables + var err error + var conditions *[]string + + if isV6 { + ipt, err = iptables.NewWithProtocol(iptables.ProtocolIPv6) + conditions = config.ConditionsV6 + } else { + ipt, err = iptables.NewWithProtocol(iptables.ProtocolIPv4) + conditions = config.ConditionsV4 + } + if err != nil { + return fmt.Errorf("failed to open iptables: %v", err) + } + + toplevelDnatChain := genToplevelDnatChain() + if err := toplevelDnatChain.setup(ipt, nil); err != nil { + return fmt.Errorf("failed to create top-level DNAT chain: %v", err) + } + + dnatChain := genDnatChain(config.Name, config.ContainerID, conditions) + _ = dnatChain.teardown(ipt) // If we somehow collide on this container ID + network, cleanup + + dnatRules := dnatRules(config.RuntimeConfig.PortMaps, containerIP) + if err := dnatChain.setup(ipt, dnatRules); err != nil { + return fmt.Errorf("unable to setup DNAT: %v", err) + } + + // Enable SNAT for connections to localhost. + // This won't work for ipv6, since the kernel doesn't have the equvalent + // route_localnet sysctl. + if *config.SNAT && !isV6 { + toplevelSnatChain := genToplevelSnatChain(isV6) + if err := toplevelSnatChain.setup(ipt, nil); err != nil { + return fmt.Errorf("failed to create top-level SNAT chain: %v", err) + } + + snatChain := genSnatChain(config.Name, config.ContainerID) + _ = snatChain.teardown(ipt) + + snatRules := snatRules(config.RuntimeConfig.PortMaps, containerIP) + if err := snatChain.setup(ipt, snatRules); err != nil { + return fmt.Errorf("unable to setup SNAT: %v", err) + } + if !isV6 { + // Set the route_localnet bit on the host interface, so that + // 127/8 can cross a routing boundary. + hostIfName := getRoutableHostIF(containerIP) + if hostIfName != "" { + if err := enableLocalnetRouting(hostIfName); err != nil { + return fmt.Errorf("unable to enable route_localnet: %v", err) + } + } + } + } + + return nil +} + +// genToplevelDnatChain creates the top-level summary chain that we'll +// add our chain to. This is easy, because creating chains is idempotent. +// IMPORTANT: do not change this, or else upgrading plugins will require +// manual intervention. +func genToplevelDnatChain() chain { + return chain{ + table: "nat", + name: TopLevelDNATChainName, + entryRule: []string{ + "-m", "addrtype", + "--dst-type", "LOCAL", + }, + entryChains: []string{"PREROUTING", "OUTPUT"}, + } +} + +// genDnatChain creates the per-container chain. +// Conditions are any static entry conditions for the chain. +func genDnatChain(netName, containerID string, conditions *[]string) chain { + name := formatChainName("DN-", netName, containerID) + comment := fmt.Sprintf(`dnat name: "%s" id: "%s"`, netName, containerID) + + ch := chain{ + table: "nat", + name: name, + entryRule: []string{ + "-m", "comment", + "--comment", comment, + }, + entryChains: []string{TopLevelDNATChainName}, + } + if conditions != nil && len(*conditions) != 0 { + ch.entryRule = append(ch.entryRule, *conditions...) + } + + return ch +} + +// dnatRules generates the destination NAT rules, one per port, to direct +// traffic from hostip:hostport to podip:podport +func dnatRules(entries []PortMapEntry, containerIP net.IP) [][]string { + out := make([][]string, 0, len(entries)) + for _, entry := range entries { + rule := []string{ + "-p", entry.Protocol, + "--dport", strconv.Itoa(entry.HostPort)} + + if entry.HostIP != "" { + rule = append(rule, + "-d", entry.HostIP) + } + + rule = append(rule, + "-j", "DNAT", + "--to-destination", fmtIpPort(containerIP, entry.ContainerPort)) + + out = append(out, rule) + } + return out +} + +// genToplevelSnatChain creates the top-level summary snat chain. +// IMPORTANT: do not change this, or else upgrading plugins will require +// manual intervention +func genToplevelSnatChain(isV6 bool) chain { + return chain{ + table: "nat", + name: TopLevelSNATChainName, + entryRule: []string{ + "-s", localhostIP(isV6), + "!", "-d", localhostIP(isV6), + }, + entryChains: []string{"POSTROUTING"}, + } +} + +// genSnatChain creates the snat (localhost) chain for this container. +func genSnatChain(netName, containerID string) chain { + name := formatChainName("SN-", netName, containerID) + comment := fmt.Sprintf(`snat name: "%s" id: "%s"`, netName, containerID) + + return chain{ + table: "nat", + name: name, + entryRule: []string{ + "-m", "comment", + "--comment", comment, + }, + entryChains: []string{TopLevelSNATChainName}, + } +} + +// snatRules sets up masquerading for connections to localhost:hostport, +// rewriting the source so that returning packets are correct. +func snatRules(entries []PortMapEntry, containerIP net.IP) [][]string { + isV6 := (containerIP.To4() == nil) + + out := make([][]string, 0, len(entries)) + for _, entry := range entries { + out = append(out, []string{ + "-p", entry.Protocol, + "-s", localhostIP(isV6), + "-d", containerIP.String(), + "--dport", strconv.Itoa(entry.ContainerPort), + "-j", "MASQUERADE", + }) + } + return out +} + +// enableLocalnetRouting tells the kernel not to treat 127/8 as a martian, +// so that connections with a source ip of 127/8 can cross a routing boundary. +func enableLocalnetRouting(ifName string) error { + routeLocalnetPath := "net.ipv4.conf." + ifName + ".route_localnet" + _, err := sysctl.Sysctl(routeLocalnetPath, "1") + return err +} + +// unforwardPorts deletes any iptables rules created by this plugin. +// It should be idempotent - it will not error if the chain does not exist. +// +// We also need to be a bit clever about how we handle errors with initializing +// iptables. We may be on a system with no ip(6)tables, or no kernel support +// for that protocol. The ADD would be successful, since it only adds forwarding +// based on the addresses assigned to the container. However, at DELETE time we +// don't know which protocols were used. +// So, we first check that iptables is "generally OK" by doing a check. If +// not, we ignore the error, unless neither v4 nor v6 are OK. +func unforwardPorts(config *PortMapConf) error { + dnatChain := genDnatChain(config.Name, config.ContainerID, nil) + snatChain := genSnatChain(config.Name, config.ContainerID) + + ip4t := maybeGetIptables(false) + ip6t := maybeGetIptables(true) + if ip4t == nil && ip6t == nil { + return fmt.Errorf("neither iptables nor ip6tables usable") + } + + if ip4t != nil { + if err := dnatChain.teardown(ip4t); err != nil { + return fmt.Errorf("could not teardown ipv4 dnat: %v", err) + } + if err := snatChain.teardown(ip4t); err != nil { + return fmt.Errorf("could not teardown ipv4 snat: %v", err) + } + } + + if ip6t != nil { + if err := dnatChain.teardown(ip6t); err != nil { + return fmt.Errorf("could not teardown ipv6 dnat: %v", err) + } + // no SNAT teardown because it doesn't work for v6 + } + return nil +} + +// maybeGetIptables implements the soft error swallowing. If iptables is +// usable for the given protocol, returns a handle, otherwise nil +func maybeGetIptables(isV6 bool) *iptables.IPTables { + proto := iptables.ProtocolIPv4 + if isV6 { + proto = iptables.ProtocolIPv6 + } + + ipt, err := iptables.NewWithProtocol(proto) + if err != nil { + return nil + } + + _, err = ipt.List("nat", "OUTPUT") + if err != nil { + return nil + } + + return ipt +} diff --git a/plugins/meta/portmap/portmap_integ_test.go b/plugins/meta/portmap/portmap_integ_test.go new file mode 100644 index 00000000..69df51c5 --- /dev/null +++ b/plugins/meta/portmap/portmap_integ_test.go @@ -0,0 +1,225 @@ +// Copyright 2017 CNI authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "fmt" + "net" + "os" + "path/filepath" + "time" + + "github.com/containernetworking/cni/libcni" + "github.com/containernetworking/cni/pkg/types/current" + "github.com/containernetworking/plugins/pkg/ns" + "github.com/coreos/go-iptables/iptables" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "github.com/vishvananda/netlink" +) + +var _ = Describe("portmap integration tests", func() { + + var configList *libcni.NetworkConfigList + var cniConf *libcni.CNIConfig + var targetNS ns.NetNS + var containerPort int + var closeChan chan interface{} + + BeforeEach(func() { + var err error + rawConfig := `{ + "cniVersion": "0.3.0", + "name": "cni-portmap-unit-test", + "plugins": [ + { + "type": "ptp", + "ipMasq": true, + "ipam": { + "type": "host-local", + "subnet": "172.16.31.0/24" + } + }, + { + "type": "portmap", + "capabilities": { + "portMappings": true + } + } + ] +}` + + configList, err = libcni.ConfListFromBytes([]byte(rawConfig)) + Expect(err).NotTo(HaveOccurred()) + + // turn PATH in to CNI_PATH + dirs := filepath.SplitList(os.Getenv("PATH")) + cniConf = &libcni.CNIConfig{Path: dirs} + + targetNS, err = ns.NewNS() + Expect(err).NotTo(HaveOccurred()) + fmt.Fprintln(GinkgoWriter, "namespace:", targetNS.Path()) + + // Start an echo server and get the port + containerPort, closeChan, err = RunEchoServerInNS(targetNS) + Expect(err).NotTo(HaveOccurred()) + + }) + + AfterEach(func() { + if targetNS != nil { + targetNS.Close() + } + }) + + // This needs to be done using Ginkgo's asynchronous testing mode. + It("forwards a TCP port on ipv4", func(done Done) { + var err error + hostPort := 9999 + runtimeConfig := libcni.RuntimeConf{ + ContainerID: "unit-test", + NetNS: targetNS.Path(), + IfName: "eth0", + CapabilityArgs: map[string]interface{}{ + "portMappings": []map[string]interface{}{ + { + "hostPort": hostPort, + "containerPort": containerPort, + "protocol": "tcp", + }, + }, + }, + } + + // Make delete idempotent, so we can clean up on failure + netDeleted := false + deleteNetwork := func() error { + if netDeleted { + return nil + } + netDeleted = true + return cniConf.DelNetworkList(configList, &runtimeConfig) + } + + // we'll also manually check the iptables chains + ipt, err := iptables.NewWithProtocol(iptables.ProtocolIPv4) + Expect(err).NotTo(HaveOccurred()) + dnatChainName := genDnatChain("cni-portmap-unit-test", "unit-test", nil).name + + // Create the network + resI, err := cniConf.AddNetworkList(configList, &runtimeConfig) + Expect(err).NotTo(HaveOccurred()) + defer deleteNetwork() + + // Check the chain exists + _, err = ipt.List("nat", dnatChainName) + Expect(err).NotTo(HaveOccurred()) + + result, err := current.GetResult(resI) + Expect(err).NotTo(HaveOccurred()) + var contIP net.IP + + for _, ip := range result.IPs { + if result.Interfaces[ip.Interface].Sandbox == "" { + continue + } + contIP = ip.Address.IP + } + if contIP == nil { + Fail("could not determine container IP") + } + + // Sanity check: verify that the container is reachable directly + contOK := testEchoServer(fmt.Sprintf("%s:%d", contIP.String(), containerPort)) + + // Verify that a connection to the forwarded port works + hostIP := getLocalIP() + dnatOK := testEchoServer(fmt.Sprintf("%s:%d", hostIP, hostPort)) + + // Verify that a connection to localhost works + snatOK := testEchoServer(fmt.Sprintf("%s:%d", "127.0.0.1", hostPort)) + + // Cleanup + close(closeChan) + err = deleteNetwork() + Expect(err).NotTo(HaveOccurred()) + + // Verify iptables rules are gone + _, err = ipt.List("nat", dnatChainName) + Expect(err).To(MatchError(ContainSubstring("iptables: No chain/target/match by that name."))) + + // Check that everything succeeded *after* we clean up the network + if !contOK { + Fail("connection direct to " + contIP.String() + " failed") + } + if !dnatOK { + Fail("Connection to " + hostIP + " was not forwarded") + } + if !snatOK { + Fail("connection to 127.0.0.1 was not forwarded") + } + + close(done) + + }, 5) +}) + +// testEchoServer returns true if we found an echo server on the port +func testEchoServer(address string) bool { + fmt.Fprintln(GinkgoWriter, "dialing", address) + conn, err := net.Dial("tcp", address) + if err != nil { + fmt.Fprintln(GinkgoWriter, "connection to", address, "failed:", err) + return false + } + defer conn.Close() + + conn.SetDeadline(time.Now().Add(2 * time.Second)) + fmt.Fprintln(GinkgoWriter, "connected to", address) + + message := "Aliquid melius quam pessimum optimum non est." + _, err = fmt.Fprint(conn, message) + if err != nil { + fmt.Fprintln(GinkgoWriter, "sending message to", address, " failed:", err) + return false + } + + conn.SetDeadline(time.Now().Add(2 * time.Second)) + fmt.Fprintln(GinkgoWriter, "reading...") + response := make([]byte, len(message)) + _, err = conn.Read(response) + if err != nil { + fmt.Fprintln(GinkgoWriter, "receiving message from", address, " failed:", err) + return false + } + + fmt.Fprintln(GinkgoWriter, "read...") + if string(response) == message { + return true + } + fmt.Fprintln(GinkgoWriter, "returned message didn't match?") + return false +} + +func getLocalIP() string { + addrs, err := netlink.AddrList(nil, netlink.FAMILY_V4) + Expect(err).NotTo(HaveOccurred()) + + for _, addr := range addrs { + return addr.IP.String() + } + Fail("no live addresses") + return "" +} diff --git a/plugins/meta/portmap/portmap_suite_test.go b/plugins/meta/portmap/portmap_suite_test.go new file mode 100644 index 00000000..51e24ff5 --- /dev/null +++ b/plugins/meta/portmap/portmap_suite_test.go @@ -0,0 +1,103 @@ +// Copyright 2017 CNI authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "fmt" + "net" + "time" + + "github.com/containernetworking/plugins/pkg/ns" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "testing" +) + +func TestPortmap(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "portmap Suite") +} + +// OpenEchoServer opens a server that handles one connection before closing. +// It opens on a random port and sends the port number on portChan when +// the server is up and running. If an error is encountered, closes portChan. +// If closeChan is closed, closes the socket. +func OpenEchoServer(portChan chan<- int, closeChan <-chan interface{}) error { + laddr, err := net.ResolveTCPAddr("tcp", "0.0.0.0:0") + if err != nil { + close(portChan) + return err + } + sock, err := net.ListenTCP("tcp", laddr) + if err != nil { + close(portChan) + return err + } + defer sock.Close() + + switch addr := sock.Addr().(type) { + case *net.TCPAddr: + portChan <- addr.Port + default: + close(portChan) + return fmt.Errorf("addr cast failed!") + } + for { + select { + case <-closeChan: + break + default: + } + + sock.SetDeadline(time.Now().Add(time.Second)) + con, err := sock.AcceptTCP() + if err != nil { + if opErr, ok := err.(*net.OpError); ok && opErr.Timeout() { + continue + } + continue + } + + buf := make([]byte, 512) + con.Read(buf) + con.Write(buf) + con.Close() + } +} + +func RunEchoServerInNS(netNS ns.NetNS) (int, chan interface{}, error) { + portChan := make(chan int) + closeChan := make(chan interface{}) + + go func() { + err := netNS.Do(func(ns.NetNS) error { + OpenEchoServer(portChan, closeChan) + return nil + }) + // Somehow the ns.Do failed + if err != nil { + close(portChan) + } + }() + + portNum := <-portChan + if portNum == 0 { + return 0, nil, fmt.Errorf("failed to execute server") + } + + return portNum, closeChan, nil +} diff --git a/plugins/meta/portmap/portmap_test.go b/plugins/meta/portmap/portmap_test.go new file mode 100644 index 00000000..6e284614 --- /dev/null +++ b/plugins/meta/portmap/portmap_test.go @@ -0,0 +1,136 @@ +// Copyright 2017 CNI authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "net" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +var _ = Describe("portmapping configuration", func() { + netName := "testNetName" + containerID := "icee6giejonei6sohng6ahngee7laquohquee9shiGo7fohferakah3Feiyoolu2pei7ciPhoh7shaoX6vai3vuf0ahfaeng8yohb9ceu0daez5hashee8ooYai5wa3y" + + mappings := []PortMapEntry{ + {80, 90, "tcp", ""}, + {1000, 2000, "udp", ""}, + } + ipv4addr := net.ParseIP("192.2.0.1") + ipv6addr := net.ParseIP("2001:db8::1") + + Describe("Generating chains", func() { + Context("for DNAT", func() { + It("generates a correct container chain", func() { + ch := genDnatChain(netName, containerID, &[]string{"-m", "hello"}) + + Expect(ch).To(Equal(chain{ + table: "nat", + name: "CNI-DN-bfd599665540dd91d5d28", + entryRule: []string{ + "-m", "comment", + "--comment", `dnat name: "testNetName" id: "` + containerID + `"`, + "-m", "hello", + }, + entryChains: []string{TopLevelDNATChainName}, + })) + }) + + It("generates a correct top-level chain", func() { + ch := genToplevelDnatChain() + + Expect(ch).To(Equal(chain{ + table: "nat", + name: "CNI-HOSTPORT-DNAT", + entryRule: []string{ + "-m", "addrtype", + "--dst-type", "LOCAL", + }, + entryChains: []string{"PREROUTING", "OUTPUT"}, + })) + }) + }) + + Context("for SNAT", func() { + It("generates a correct container chain", func() { + ch := genSnatChain(netName, containerID) + + Expect(ch).To(Equal(chain{ + table: "nat", + name: "CNI-SN-bfd599665540dd91d5d28", + entryRule: []string{ + "-m", "comment", + "--comment", `snat name: "testNetName" id: "` + containerID + `"`, + }, + entryChains: []string{TopLevelSNATChainName}, + })) + }) + + It("generates a correct top-level chain", func() { + Context("for ipv4", func() { + ch := genToplevelSnatChain(false) + Expect(ch).To(Equal(chain{ + table: "nat", + name: "CNI-HOSTPORT-SNAT", + entryRule: []string{ + "-s", "127.0.0.1", + "!", "-d", "127.0.0.1", + }, + entryChains: []string{"POSTROUTING"}, + })) + }) + }) + }) + }) + + Describe("Forwarding rules", func() { + Context("for DNAT", func() { + It("generates correct ipv4 rules", func() { + rules := dnatRules(mappings, ipv4addr) + Expect(rules).To(Equal([][]string{ + {"-p", "tcp", "--dport", "80", "-j", "DNAT", "--to-destination", "192.2.0.1:90"}, + {"-p", "udp", "--dport", "1000", "-j", "DNAT", "--to-destination", "192.2.0.1:2000"}, + })) + }) + It("generates correct ipv6 rules", func() { + rules := dnatRules(mappings, ipv6addr) + Expect(rules).To(Equal([][]string{ + {"-p", "tcp", "--dport", "80", "-j", "DNAT", "--to-destination", "[2001:db8::1]:90"}, + {"-p", "udp", "--dport", "1000", "-j", "DNAT", "--to-destination", "[2001:db8::1]:2000"}, + })) + }) + }) + + Context("for SNAT", func() { + + It("generates correct ipv4 rules", func() { + rules := snatRules(mappings, ipv4addr) + Expect(rules).To(Equal([][]string{ + {"-p", "tcp", "-s", "127.0.0.1", "-d", "192.2.0.1", "--dport", "90", "-j", "MASQUERADE"}, + {"-p", "udp", "-s", "127.0.0.1", "-d", "192.2.0.1", "--dport", "2000", "-j", "MASQUERADE"}, + })) + }) + + It("generates correct ipv6 rules", func() { + rules := snatRules(mappings, ipv6addr) + Expect(rules).To(Equal([][]string{ + {"-p", "tcp", "-s", "::1", "-d", "2001:db8::1", "--dport", "90", "-j", "MASQUERADE"}, + {"-p", "udp", "-s", "::1", "-d", "2001:db8::1", "--dport", "2000", "-j", "MASQUERADE"}, + })) + }) + }) + }) +}) diff --git a/plugins/meta/portmap/utils.go b/plugins/meta/portmap/utils.go new file mode 100644 index 00000000..a0c9b33b --- /dev/null +++ b/plugins/meta/portmap/utils.go @@ -0,0 +1,67 @@ +// Copyright 2017 CNI authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "crypto/sha512" + "fmt" + "net" + + "github.com/vishvananda/netlink" +) + +const maxChainNameLength = 28 + +// fmtIpPort correctly formats ip:port literals for iptables and ip6tables - +// need to wrap v6 literals in a [] +func fmtIpPort(ip net.IP, port int) string { + if ip.To4() == nil { + return fmt.Sprintf("[%s]:%d", ip.String(), port) + } + return fmt.Sprintf("%s:%d", ip.String(), port) +} + +func localhostIP(isV6 bool) string { + if isV6 { + return "::1" + } + return "127.0.0.1" +} + +// getRoutableHostIF will try and determine which interface routes the container's +// traffic. This is the one on which we disable martian filtering. +func getRoutableHostIF(containerIP net.IP) string { + routes, err := netlink.RouteGet(containerIP) + if err != nil { + return "" + } + + for _, route := range routes { + link, err := netlink.LinkByIndex(route.LinkIndex) + if err != nil { + continue + } + + return link.Attrs().Name + } + + return "" +} + +func formatChainName(prefix, name, id string) string { + chainBytes := sha512.Sum512([]byte(name + id)) + chain := fmt.Sprintf("CNI-%s%x", prefix, chainBytes) + return chain[:maxChainNameLength] +} diff --git a/test.sh b/test.sh index 06888aa8..0300f34d 100755 --- a/test.sh +++ b/test.sh @@ -10,7 +10,7 @@ source ./build.sh echo "Running tests" -TESTABLE="plugins/ipam/dhcp plugins/ipam/host-local plugins/ipam/host-local/backend/allocator plugins/main/loopback plugins/main/ipvlan plugins/main/macvlan plugins/main/bridge plugins/main/ptp plugins/meta/flannel plugins/main/vlan plugins/sample pkg/ip pkg/ipam pkg/ns pkg/utils pkg/utils/hwaddr pkg/utils/sysctl" +TESTABLE="plugins/ipam/dhcp plugins/ipam/host-local plugins/ipam/host-local/backend/allocator plugins/main/loopback plugins/main/ipvlan plugins/main/macvlan plugins/main/bridge plugins/main/ptp plugins/meta/flannel plugins/main/vlan plugins/sample pkg/ip pkg/ipam pkg/ns pkg/utils pkg/utils/hwaddr pkg/utils/sysctl plugins/meta/portmap" # user has not provided PKG override if [ -z "$PKG" ]; then