From a3ccebc6ecda96547e27154827d57e9989979543 Mon Sep 17 00:00:00 2001 From: Dan Winship Date: Sat, 22 Jul 2023 11:12:40 -0400 Subject: [PATCH] Add a backend abstraction to the portmap plugin Signed-off-by: Dan Winship --- plugins/meta/portmap/main.go | 49 +++- .../{portmap.go => portmap_iptables.go} | 28 +- plugins/meta/portmap/portmap_iptables_test.go | 252 ++++++++++++++++++ plugins/meta/portmap/portmap_test.go | 220 --------------- plugins/meta/portmap/utils.go | 10 + 5 files changed, 304 insertions(+), 255 deletions(-) rename plugins/meta/portmap/{portmap.go => portmap_iptables.go} (93%) create mode 100644 plugins/meta/portmap/portmap_iptables_test.go diff --git a/plugins/meta/portmap/main.go b/plugins/meta/portmap/main.go index 108c1f59..a654e0f6 100644 --- a/plugins/meta/portmap/main.go +++ b/plugins/meta/portmap/main.go @@ -40,6 +40,12 @@ import ( bv "github.com/containernetworking/plugins/pkg/utils/buildversion" ) +type PortMapper interface { + forwardPorts(config *PortMapConf, containerNet net.IPNet) error + checkPorts(config *PortMapConf, containerNet net.IPNet) error + unforwardPorts(config *PortMapConf) error +} + // PortMapEntry corresponds to a single entry in the port_mappings argument, // see CONVENTIONS.md type PortMapEntry struct { @@ -51,16 +57,22 @@ type PortMapEntry struct { type PortMapConf struct { types.NetConf - SNAT *bool `json:"snat,omitempty"` - ConditionsV4 *[]string `json:"conditionsV4"` - ConditionsV6 *[]string `json:"conditionsV6"` - MasqAll bool `json:"masqAll,omitempty"` - MarkMasqBit *int `json:"markMasqBit"` - ExternalSetMarkChain *string `json:"externalSetMarkChain"` - RuntimeConfig struct { + + mapper PortMapper + + // Generic config + SNAT *bool `json:"snat,omitempty"` + ConditionsV4 *[]string `json:"conditionsV4"` + ConditionsV6 *[]string `json:"conditionsV6"` + MasqAll bool `json:"masqAll,omitempty"` + MarkMasqBit *int `json:"markMasqBit"` + RuntimeConfig struct { PortMaps []PortMapEntry `json:"portMappings,omitempty"` } `json:"runtimeConfig,omitempty"` + // iptables-backend-specific config + ExternalSetMarkChain *string `json:"externalSetMarkChain"` + // These are fields parsed out of the config or the environment; // included here for convenience ContainerID string `json:"-"` @@ -89,7 +101,7 @@ func cmdAdd(args *skel.CmdArgs) error { netConf.ContainerID = args.ContainerID if netConf.ContIPv4.IP != nil { - if err := forwardPorts(netConf, netConf.ContIPv4); err != nil { + if err := netConf.mapper.forwardPorts(netConf, netConf.ContIPv4); err != nil { return err } // Delete conntrack entries for UDP to avoid conntrack blackholing traffic @@ -98,10 +110,21 @@ func cmdAdd(args *skel.CmdArgs) error { if err := deletePortmapStaleConnections(netConf.RuntimeConfig.PortMaps, unix.AF_INET); err != nil { log.Printf("failed to delete stale UDP conntrack entries for %s: %v", netConf.ContIPv4.IP, err) } + + if *netConf.SNAT { + // Set the route_localnet bit on the host interface, so that + // 127/8 can cross a routing boundary. + hostIfName := getRoutableHostIF(netConf.ContIPv4.IP) + if hostIfName != "" { + if err := enableLocalnetRouting(hostIfName); err != nil { + return fmt.Errorf("unable to enable route_localnet: %v", err) + } + } + } } if netConf.ContIPv6.IP != nil { - if err := forwardPorts(netConf, netConf.ContIPv6); err != nil { + if err := netConf.mapper.forwardPorts(netConf, netConf.ContIPv6); err != nil { return err } // Delete conntrack entries for UDP to avoid conntrack blackholing traffic @@ -130,7 +153,7 @@ func cmdDel(args *skel.CmdArgs) error { // We don't need to parse out whether or not we're using v6 or snat, // deletion is idempotent - return unforwardPorts(netConf) + return netConf.mapper.unforwardPorts(netConf) } func main() { @@ -161,13 +184,13 @@ func cmdCheck(args *skel.CmdArgs) error { conf.ContainerID = args.ContainerID if conf.ContIPv4.IP != nil { - if err := checkPorts(conf, conf.ContIPv4); err != nil { + if err := conf.mapper.checkPorts(conf, conf.ContIPv4); err != nil { return err } } if conf.ContIPv6.IP != nil { - if err := checkPorts(conf, conf.ContIPv6); err != nil { + if err := conf.mapper.checkPorts(conf, conf.ContIPv6); err != nil { return err } } @@ -197,6 +220,8 @@ func parseConfig(stdin []byte, ifName string) (*PortMapConf, *current.Result, er } } + conf.mapper = &portMapperIPTables{} + if conf.SNAT == nil { tvar := true conf.SNAT = &tvar diff --git a/plugins/meta/portmap/portmap.go b/plugins/meta/portmap/portmap_iptables.go similarity index 93% rename from plugins/meta/portmap/portmap.go rename to plugins/meta/portmap/portmap_iptables.go index e380da93..07d6c420 100644 --- a/plugins/meta/portmap/portmap.go +++ b/plugins/meta/portmap/portmap_iptables.go @@ -25,7 +25,6 @@ import ( "github.com/vishvananda/netlink" "github.com/containernetworking/plugins/pkg/utils" - "github.com/containernetworking/plugins/pkg/utils/sysctl" ) // This creates the chains to be added to iptables. The basic structure is @@ -52,9 +51,11 @@ const ( OldTopLevelSNATChainName = "CNI-HOSTPORT-SNAT" ) +type portMapperIPTables struct{} + // forwardPorts establishes port forwarding to a given container IP. // containerNet.IP can be either v4 or v6. -func forwardPorts(config *PortMapConf, containerNet net.IPNet) error { +func (*portMapperIPTables) forwardPorts(config *PortMapConf, containerNet net.IPNet) error { isV6 := (containerNet.IP.To4() == nil) var ipt *iptables.IPTables @@ -87,17 +88,6 @@ func forwardPorts(config *PortMapConf, containerNet net.IPNet) error { return fmt.Errorf("unable to create chain %s: %v", setMarkChain.name, err) } } - - if !isV6 { - // Set the route_localnet bit on the host interface, so that - // 127/8 can cross a routing boundary. - hostIfName := getRoutableHostIF(containerNet.IP) - if hostIfName != "" { - if err := enableLocalnetRouting(hostIfName); err != nil { - return fmt.Errorf("unable to enable route_localnet: %v", err) - } - } - } } // Generate the DNAT (actual port forwarding) rules @@ -117,7 +107,7 @@ func forwardPorts(config *PortMapConf, containerNet net.IPNet) error { return nil } -func checkPorts(config *PortMapConf, containerNet net.IPNet) error { +func (*portMapperIPTables) checkPorts(config *PortMapConf, containerNet net.IPNet) error { isV6 := (containerNet.IP.To4() == nil) dnatChain := genDnatChain(config.Name, config.ContainerID) fillDnatRules(&dnatChain, config, containerNet) @@ -344,14 +334,6 @@ func genMarkMasqChain(markBit int) chain { return ch } -// enableLocalnetRouting tells the kernel not to treat 127/8 as a martian, -// so that connections with a source ip of 127/8 can cross a routing boundary. -func enableLocalnetRouting(ifName string) error { - routeLocalnetPath := "net/ipv4/conf/" + ifName + "/route_localnet" - _, err := sysctl.Sysctl(routeLocalnetPath, "1") - return err -} - // genOldSnatChain is no longer used, but used to be created. We'll try and // tear it down in case the plugin version changed between ADD and DEL func genOldSnatChain(netName, containerID string) chain { @@ -372,7 +354,7 @@ func genOldSnatChain(netName, containerID string) chain { // don't know which protocols were used. // So, we first check that iptables is "generally OK" by doing a check. If // not, we ignore the error, unless neither v4 nor v6 are OK. -func unforwardPorts(config *PortMapConf) error { +func (*portMapperIPTables) unforwardPorts(config *PortMapConf) error { dnatChain := genDnatChain(config.Name, config.ContainerID) // Might be lying around from old versions diff --git a/plugins/meta/portmap/portmap_iptables_test.go b/plugins/meta/portmap/portmap_iptables_test.go new file mode 100644 index 00000000..bc9bbf22 --- /dev/null +++ b/plugins/meta/portmap/portmap_iptables_test.go @@ -0,0 +1,252 @@ +// Copyright 2017 CNI authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "fmt" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/containernetworking/cni/pkg/types" +) + +var _ = Describe("portmapping configuration (iptables)", func() { + netName := "testNetName" + containerID := "icee6giejonei6sohng6ahngee7laquohquee9shiGo7fohferakah3Feiyoolu2pei7ciPhoh7shaoX6vai3vuf0ahfaeng8yohb9ceu0daez5hashee8ooYai5wa3y" + + for _, ver := range []string{"0.3.0", "0.3.1", "0.4.0", "1.0.0"} { + // Redefine ver inside for scope so real value is picked up by each dynamically defined It() + // See Gingkgo's "Patterns for dynamically generating tests" documentation. + ver := ver + + Describe("Generating iptables chains", func() { + Context("for DNAT", func() { + It(fmt.Sprintf("[%s] generates a correct standard container chain", ver), func() { + ch := genDnatChain(netName, containerID) + + Expect(ch).To(Equal(chain{ + table: "nat", + name: "CNI-DN-bfd599665540dd91d5d28", + entryChains: []string{TopLevelDNATChainName}, + })) + configBytes := []byte(fmt.Sprintf(`{ + "name": "test", + "type": "portmap", + "cniVersion": "%s", + "runtimeConfig": { + "portMappings": [ + { "hostPort": 8080, "containerPort": 80, "protocol": "tcp"}, + { "hostPort": 8081, "containerPort": 80, "protocol": "tcp"}, + { "hostPort": 8080, "containerPort": 81, "protocol": "udp"}, + { "hostPort": 8082, "containerPort": 82, "protocol": "udp"}, + { "hostPort": 8083, "containerPort": 83, "protocol": "tcp", "hostIP": "192.168.0.2"}, + { "hostPort": 8084, "containerPort": 84, "protocol": "tcp", "hostIP": "0.0.0.0"}, + { "hostPort": 8085, "containerPort": 85, "protocol": "tcp", "hostIP": "2001:db8:a::1"}, + { "hostPort": 8086, "containerPort": 86, "protocol": "tcp", "hostIP": "::"} + ] + }, + "snat": true, + "conditionsV4": ["-a", "b"], + "conditionsV6": ["-c", "d"] + }`, ver)) + + conf, _, err := parseConfig(configBytes, "foo") + Expect(err).NotTo(HaveOccurred()) + conf.ContainerID = containerID + + ch = genDnatChain(conf.Name, containerID) + Expect(ch).To(Equal(chain{ + table: "nat", + name: "CNI-DN-67e92b96e692a494b6b85", + entryChains: []string{"CNI-HOSTPORT-DNAT"}, + })) + + n, err := types.ParseCIDR("10.0.0.2/24") + Expect(err).NotTo(HaveOccurred()) + fillDnatRules(&ch, conf, *n) + + Expect(ch.entryRules).To(Equal([][]string{ + { + "-m", "comment", "--comment", + fmt.Sprintf("dnat name: \"test\" id: \"%s\"", containerID), + "-m", "multiport", + "-p", "tcp", + "--destination-ports", "8080,8081,8083,8084,8085,8086", + "-a", "b", + }, + { + "-m", "comment", "--comment", + fmt.Sprintf("dnat name: \"test\" id: \"%s\"", containerID), + "-m", "multiport", + "-p", "udp", + "--destination-ports", "8080,8082", + "-a", "b", + }, + })) + + Expect(ch.rules).To(Equal([][]string{ + // tcp rules and not hostIP + {"-p", "tcp", "--dport", "8080", "-s", "10.0.0.2/24", "-j", "CNI-HOSTPORT-SETMARK"}, + {"-p", "tcp", "--dport", "8080", "-s", "127.0.0.1", "-j", "CNI-HOSTPORT-SETMARK"}, + {"-p", "tcp", "--dport", "8080", "-j", "DNAT", "--to-destination", "10.0.0.2:80"}, + {"-p", "tcp", "--dport", "8081", "-s", "10.0.0.2/24", "-j", "CNI-HOSTPORT-SETMARK"}, + {"-p", "tcp", "--dport", "8081", "-s", "127.0.0.1", "-j", "CNI-HOSTPORT-SETMARK"}, + {"-p", "tcp", "--dport", "8081", "-j", "DNAT", "--to-destination", "10.0.0.2:80"}, + // udp rules and not hostIP + {"-p", "udp", "--dport", "8080", "-s", "10.0.0.2/24", "-j", "CNI-HOSTPORT-SETMARK"}, + {"-p", "udp", "--dport", "8080", "-s", "127.0.0.1", "-j", "CNI-HOSTPORT-SETMARK"}, + {"-p", "udp", "--dport", "8080", "-j", "DNAT", "--to-destination", "10.0.0.2:81"}, + {"-p", "udp", "--dport", "8082", "-s", "10.0.0.2/24", "-j", "CNI-HOSTPORT-SETMARK"}, + {"-p", "udp", "--dport", "8082", "-s", "127.0.0.1", "-j", "CNI-HOSTPORT-SETMARK"}, + {"-p", "udp", "--dport", "8082", "-j", "DNAT", "--to-destination", "10.0.0.2:82"}, + // tcp rules and hostIP + {"-p", "tcp", "--dport", "8083", "-d", "192.168.0.2", "-s", "10.0.0.2/24", "-j", "CNI-HOSTPORT-SETMARK"}, + {"-p", "tcp", "--dport", "8083", "-d", "192.168.0.2", "-s", "127.0.0.1", "-j", "CNI-HOSTPORT-SETMARK"}, + {"-p", "tcp", "--dport", "8083", "-d", "192.168.0.2", "-j", "DNAT", "--to-destination", "10.0.0.2:83"}, + // tcp rules and hostIP = "0.0.0.0" + {"-p", "tcp", "--dport", "8084", "-s", "10.0.0.2/24", "-j", "CNI-HOSTPORT-SETMARK"}, + {"-p", "tcp", "--dport", "8084", "-s", "127.0.0.1", "-j", "CNI-HOSTPORT-SETMARK"}, + {"-p", "tcp", "--dport", "8084", "-j", "DNAT", "--to-destination", "10.0.0.2:84"}, + })) + + ch.rules = nil + ch.entryRules = nil + + n, err = types.ParseCIDR("2001:db8::2/64") + Expect(err).NotTo(HaveOccurred()) + fillDnatRules(&ch, conf, *n) + + Expect(ch.rules).To(Equal([][]string{ + // tcp rules and not hostIP + {"-p", "tcp", "--dport", "8080", "-s", "2001:db8::2/64", "-j", "CNI-HOSTPORT-SETMARK"}, + {"-p", "tcp", "--dport", "8080", "-j", "DNAT", "--to-destination", "[2001:db8::2]:80"}, + {"-p", "tcp", "--dport", "8081", "-s", "2001:db8::2/64", "-j", "CNI-HOSTPORT-SETMARK"}, + {"-p", "tcp", "--dport", "8081", "-j", "DNAT", "--to-destination", "[2001:db8::2]:80"}, + // udp rules and not hostIP + {"-p", "udp", "--dport", "8080", "-s", "2001:db8::2/64", "-j", "CNI-HOSTPORT-SETMARK"}, + {"-p", "udp", "--dport", "8080", "-j", "DNAT", "--to-destination", "[2001:db8::2]:81"}, + {"-p", "udp", "--dport", "8082", "-s", "2001:db8::2/64", "-j", "CNI-HOSTPORT-SETMARK"}, + {"-p", "udp", "--dport", "8082", "-j", "DNAT", "--to-destination", "[2001:db8::2]:82"}, + // tcp rules and hostIP + {"-p", "tcp", "--dport", "8085", "-d", "2001:db8:a::1", "-s", "2001:db8::2/64", "-j", "CNI-HOSTPORT-SETMARK"}, + {"-p", "tcp", "--dport", "8085", "-d", "2001:db8:a::1", "-j", "DNAT", "--to-destination", "[2001:db8::2]:85"}, + // tcp rules and hostIP = "::" + {"-p", "tcp", "--dport", "8086", "-s", "2001:db8::2/64", "-j", "CNI-HOSTPORT-SETMARK"}, + {"-p", "tcp", "--dport", "8086", "-j", "DNAT", "--to-destination", "[2001:db8::2]:86"}, + })) + + // Disable snat, generate rules + ch.rules = nil + ch.entryRules = nil + fvar := false + conf.SNAT = &fvar + + n, err = types.ParseCIDR("10.0.0.2/24") + Expect(err).NotTo(HaveOccurred()) + fillDnatRules(&ch, conf, *n) + Expect(ch.rules).To(Equal([][]string{ + {"-p", "tcp", "--dport", "8080", "-j", "DNAT", "--to-destination", "10.0.0.2:80"}, + {"-p", "tcp", "--dport", "8081", "-j", "DNAT", "--to-destination", "10.0.0.2:80"}, + {"-p", "udp", "--dport", "8080", "-j", "DNAT", "--to-destination", "10.0.0.2:81"}, + {"-p", "udp", "--dport", "8082", "-j", "DNAT", "--to-destination", "10.0.0.2:82"}, + {"-p", "tcp", "--dport", "8083", "-d", "192.168.0.2", "-j", "DNAT", "--to-destination", "10.0.0.2:83"}, + {"-p", "tcp", "--dport", "8084", "-j", "DNAT", "--to-destination", "10.0.0.2:84"}, + })) + }) + + It(fmt.Sprintf("[%s] generates a correct chain with external mark", ver), func() { + ch := genDnatChain(netName, containerID) + + Expect(ch).To(Equal(chain{ + table: "nat", + name: "CNI-DN-bfd599665540dd91d5d28", + entryChains: []string{TopLevelDNATChainName}, + })) + configBytes := []byte(fmt.Sprintf(`{ + "name": "test", + "type": "portmap", + "cniVersion": "%s", + "runtimeConfig": { + "portMappings": [ + { "hostPort": 8080, "containerPort": 80, "protocol": "tcp"} + ] + }, + "externalSetMarkChain": "PLZ-SET-MARK", + "conditionsV4": ["-a", "b"], + "conditionsV6": ["-c", "d"] + }`, ver)) + + conf, _, err := parseConfig(configBytes, "foo") + Expect(err).NotTo(HaveOccurred()) + conf.ContainerID = containerID + + ch = genDnatChain(conf.Name, containerID) + n, err := types.ParseCIDR("10.0.0.2/24") + Expect(err).NotTo(HaveOccurred()) + fillDnatRules(&ch, conf, *n) + Expect(ch.rules).To(Equal([][]string{ + {"-p", "tcp", "--dport", "8080", "-s", "10.0.0.2/24", "-j", "PLZ-SET-MARK"}, + {"-p", "tcp", "--dport", "8080", "-s", "127.0.0.1", "-j", "PLZ-SET-MARK"}, + {"-p", "tcp", "--dport", "8080", "-j", "DNAT", "--to-destination", "10.0.0.2:80"}, + })) + }) + + It(fmt.Sprintf("[%s] generates a correct top-level chain", ver), func() { + ch := genToplevelDnatChain() + + Expect(ch).To(Equal(chain{ + table: "nat", + name: "CNI-HOSTPORT-DNAT", + entryChains: []string{"PREROUTING", "OUTPUT"}, + entryRules: [][]string{{"-m", "addrtype", "--dst-type", "LOCAL"}}, + })) + }) + + It(fmt.Sprintf("[%s] generates the correct mark chains", ver), func() { + masqBit := 5 + ch := genSetMarkChain(masqBit) + Expect(ch).To(Equal(chain{ + table: "nat", + name: "CNI-HOSTPORT-SETMARK", + rules: [][]string{{ + "-m", "comment", + "--comment", "CNI portfwd masquerade mark", + "-j", "MARK", + "--set-xmark", "0x20/0x20", + }}, + })) + + ch = genMarkMasqChain(masqBit) + Expect(ch).To(Equal(chain{ + table: "nat", + name: "CNI-HOSTPORT-MASQ", + entryChains: []string{"POSTROUTING"}, + entryRules: [][]string{{ + "-m", "comment", + "--comment", "CNI portfwd requiring masquerade", + }}, + rules: [][]string{{ + "-m", "mark", + "--mark", "0x20/0x20", + "-j", "MASQUERADE", + }}, + prependEntry: true, + })) + }) + }) + }) + } +}) diff --git a/plugins/meta/portmap/portmap_test.go b/plugins/meta/portmap/portmap_test.go index 4897fe63..fbb69b39 100644 --- a/plugins/meta/portmap/portmap_test.go +++ b/plugins/meta/portmap/portmap_test.go @@ -24,9 +24,6 @@ import ( ) var _ = Describe("portmapping configuration", func() { - netName := "testNetName" - containerID := "icee6giejonei6sohng6ahngee7laquohquee9shiGo7fohferakah3Feiyoolu2pei7ciPhoh7shaoX6vai3vuf0ahfaeng8yohb9ceu0daez5hashee8ooYai5wa3y" - for _, ver := range []string{"0.3.0", "0.3.1", "0.4.0", "1.0.0"} { // Redefine ver inside for scope so real value is picked up by each dynamically defined It() // See Gingkgo's "Patterns for dynamically generating tests" documentation. @@ -157,222 +154,5 @@ var _ = Describe("portmapping configuration", func() { Expect(err).NotTo(HaveOccurred()) }) }) - - Describe("Generating chains", func() { - Context("for DNAT", func() { - It(fmt.Sprintf("[%s] generates a correct standard container chain", ver), func() { - ch := genDnatChain(netName, containerID) - - Expect(ch).To(Equal(chain{ - table: "nat", - name: "CNI-DN-bfd599665540dd91d5d28", - entryChains: []string{TopLevelDNATChainName}, - })) - configBytes := []byte(fmt.Sprintf(`{ - "name": "test", - "type": "portmap", - "cniVersion": "%s", - "runtimeConfig": { - "portMappings": [ - { "hostPort": 8080, "containerPort": 80, "protocol": "tcp"}, - { "hostPort": 8081, "containerPort": 80, "protocol": "tcp"}, - { "hostPort": 8080, "containerPort": 81, "protocol": "udp"}, - { "hostPort": 8082, "containerPort": 82, "protocol": "udp"}, - { "hostPort": 8083, "containerPort": 83, "protocol": "tcp", "hostIP": "192.168.0.2"}, - { "hostPort": 8084, "containerPort": 84, "protocol": "tcp", "hostIP": "0.0.0.0"}, - { "hostPort": 8085, "containerPort": 85, "protocol": "tcp", "hostIP": "2001:db8:a::1"}, - { "hostPort": 8086, "containerPort": 86, "protocol": "tcp", "hostIP": "::"} - ] - }, - "snat": true, - "conditionsV4": ["a", "b"], - "conditionsV6": ["c", "d"] - }`, ver)) - - conf, _, err := parseConfig(configBytes, "foo") - Expect(err).NotTo(HaveOccurred()) - conf.ContainerID = containerID - - ch = genDnatChain(conf.Name, containerID) - Expect(ch).To(Equal(chain{ - table: "nat", - name: "CNI-DN-67e92b96e692a494b6b85", - entryChains: []string{"CNI-HOSTPORT-DNAT"}, - })) - - n, err := types.ParseCIDR("10.0.0.2/24") - Expect(err).NotTo(HaveOccurred()) - fillDnatRules(&ch, conf, *n) - - Expect(ch.entryRules).To(Equal([][]string{ - { - "-m", "comment", "--comment", - fmt.Sprintf("dnat name: \"test\" id: \"%s\"", containerID), - "-m", "multiport", - "-p", "tcp", - "--destination-ports", "8080,8081,8083,8084,8085,8086", - "a", "b", - }, - { - "-m", "comment", "--comment", - fmt.Sprintf("dnat name: \"test\" id: \"%s\"", containerID), - "-m", "multiport", - "-p", "udp", - "--destination-ports", "8080,8082", - "a", "b", - }, - })) - - Expect(ch.rules).To(Equal([][]string{ - // tcp rules and not hostIP - {"-p", "tcp", "--dport", "8080", "-s", "10.0.0.2/24", "-j", "CNI-HOSTPORT-SETMARK"}, - {"-p", "tcp", "--dport", "8080", "-s", "127.0.0.1", "-j", "CNI-HOSTPORT-SETMARK"}, - {"-p", "tcp", "--dport", "8080", "-j", "DNAT", "--to-destination", "10.0.0.2:80"}, - {"-p", "tcp", "--dport", "8081", "-s", "10.0.0.2/24", "-j", "CNI-HOSTPORT-SETMARK"}, - {"-p", "tcp", "--dport", "8081", "-s", "127.0.0.1", "-j", "CNI-HOSTPORT-SETMARK"}, - {"-p", "tcp", "--dport", "8081", "-j", "DNAT", "--to-destination", "10.0.0.2:80"}, - // udp rules and not hostIP - {"-p", "udp", "--dport", "8080", "-s", "10.0.0.2/24", "-j", "CNI-HOSTPORT-SETMARK"}, - {"-p", "udp", "--dport", "8080", "-s", "127.0.0.1", "-j", "CNI-HOSTPORT-SETMARK"}, - {"-p", "udp", "--dport", "8080", "-j", "DNAT", "--to-destination", "10.0.0.2:81"}, - {"-p", "udp", "--dport", "8082", "-s", "10.0.0.2/24", "-j", "CNI-HOSTPORT-SETMARK"}, - {"-p", "udp", "--dport", "8082", "-s", "127.0.0.1", "-j", "CNI-HOSTPORT-SETMARK"}, - {"-p", "udp", "--dport", "8082", "-j", "DNAT", "--to-destination", "10.0.0.2:82"}, - // tcp rules and hostIP - {"-p", "tcp", "--dport", "8083", "-d", "192.168.0.2", "-s", "10.0.0.2/24", "-j", "CNI-HOSTPORT-SETMARK"}, - {"-p", "tcp", "--dport", "8083", "-d", "192.168.0.2", "-s", "127.0.0.1", "-j", "CNI-HOSTPORT-SETMARK"}, - {"-p", "tcp", "--dport", "8083", "-d", "192.168.0.2", "-j", "DNAT", "--to-destination", "10.0.0.2:83"}, - // tcp rules and hostIP = "0.0.0.0" - {"-p", "tcp", "--dport", "8084", "-s", "10.0.0.2/24", "-j", "CNI-HOSTPORT-SETMARK"}, - {"-p", "tcp", "--dport", "8084", "-s", "127.0.0.1", "-j", "CNI-HOSTPORT-SETMARK"}, - {"-p", "tcp", "--dport", "8084", "-j", "DNAT", "--to-destination", "10.0.0.2:84"}, - })) - - ch.rules = nil - ch.entryRules = nil - - n, err = types.ParseCIDR("2001:db8::2/64") - Expect(err).NotTo(HaveOccurred()) - fillDnatRules(&ch, conf, *n) - - Expect(ch.rules).To(Equal([][]string{ - // tcp rules and not hostIP - {"-p", "tcp", "--dport", "8080", "-s", "2001:db8::2/64", "-j", "CNI-HOSTPORT-SETMARK"}, - {"-p", "tcp", "--dport", "8080", "-j", "DNAT", "--to-destination", "[2001:db8::2]:80"}, - {"-p", "tcp", "--dport", "8081", "-s", "2001:db8::2/64", "-j", "CNI-HOSTPORT-SETMARK"}, - {"-p", "tcp", "--dport", "8081", "-j", "DNAT", "--to-destination", "[2001:db8::2]:80"}, - // udp rules and not hostIP - {"-p", "udp", "--dport", "8080", "-s", "2001:db8::2/64", "-j", "CNI-HOSTPORT-SETMARK"}, - {"-p", "udp", "--dport", "8080", "-j", "DNAT", "--to-destination", "[2001:db8::2]:81"}, - {"-p", "udp", "--dport", "8082", "-s", "2001:db8::2/64", "-j", "CNI-HOSTPORT-SETMARK"}, - {"-p", "udp", "--dport", "8082", "-j", "DNAT", "--to-destination", "[2001:db8::2]:82"}, - // tcp rules and hostIP - {"-p", "tcp", "--dport", "8085", "-d", "2001:db8:a::1", "-s", "2001:db8::2/64", "-j", "CNI-HOSTPORT-SETMARK"}, - {"-p", "tcp", "--dport", "8085", "-d", "2001:db8:a::1", "-j", "DNAT", "--to-destination", "[2001:db8::2]:85"}, - // tcp rules and hostIP = "::" - {"-p", "tcp", "--dport", "8086", "-s", "2001:db8::2/64", "-j", "CNI-HOSTPORT-SETMARK"}, - {"-p", "tcp", "--dport", "8086", "-j", "DNAT", "--to-destination", "[2001:db8::2]:86"}, - })) - - // Disable snat, generate rules - ch.rules = nil - ch.entryRules = nil - fvar := false - conf.SNAT = &fvar - - n, err = types.ParseCIDR("10.0.0.2/24") - Expect(err).NotTo(HaveOccurred()) - fillDnatRules(&ch, conf, *n) - Expect(ch.rules).To(Equal([][]string{ - {"-p", "tcp", "--dport", "8080", "-j", "DNAT", "--to-destination", "10.0.0.2:80"}, - {"-p", "tcp", "--dport", "8081", "-j", "DNAT", "--to-destination", "10.0.0.2:80"}, - {"-p", "udp", "--dport", "8080", "-j", "DNAT", "--to-destination", "10.0.0.2:81"}, - {"-p", "udp", "--dport", "8082", "-j", "DNAT", "--to-destination", "10.0.0.2:82"}, - {"-p", "tcp", "--dport", "8083", "-d", "192.168.0.2", "-j", "DNAT", "--to-destination", "10.0.0.2:83"}, - {"-p", "tcp", "--dport", "8084", "-j", "DNAT", "--to-destination", "10.0.0.2:84"}, - })) - }) - - It(fmt.Sprintf("[%s] generates a correct chain with external mark", ver), func() { - ch := genDnatChain(netName, containerID) - - Expect(ch).To(Equal(chain{ - table: "nat", - name: "CNI-DN-bfd599665540dd91d5d28", - entryChains: []string{TopLevelDNATChainName}, - })) - configBytes := []byte(fmt.Sprintf(`{ - "name": "test", - "type": "portmap", - "cniVersion": "%s", - "runtimeConfig": { - "portMappings": [ - { "hostPort": 8080, "containerPort": 80, "protocol": "tcp"} - ] - }, - "externalSetMarkChain": "PLZ-SET-MARK", - "conditionsV4": ["a", "b"], - "conditionsV6": ["c", "d"] - }`, ver)) - - conf, _, err := parseConfig(configBytes, "foo") - Expect(err).NotTo(HaveOccurred()) - conf.ContainerID = containerID - - ch = genDnatChain(conf.Name, containerID) - n, err := types.ParseCIDR("10.0.0.2/24") - Expect(err).NotTo(HaveOccurred()) - fillDnatRules(&ch, conf, *n) - Expect(ch.rules).To(Equal([][]string{ - {"-p", "tcp", "--dport", "8080", "-s", "10.0.0.2/24", "-j", "PLZ-SET-MARK"}, - {"-p", "tcp", "--dport", "8080", "-s", "127.0.0.1", "-j", "PLZ-SET-MARK"}, - {"-p", "tcp", "--dport", "8080", "-j", "DNAT", "--to-destination", "10.0.0.2:80"}, - })) - }) - - It(fmt.Sprintf("[%s] generates a correct top-level chain", ver), func() { - ch := genToplevelDnatChain() - - Expect(ch).To(Equal(chain{ - table: "nat", - name: "CNI-HOSTPORT-DNAT", - entryChains: []string{"PREROUTING", "OUTPUT"}, - entryRules: [][]string{{"-m", "addrtype", "--dst-type", "LOCAL"}}, - })) - }) - - It(fmt.Sprintf("[%s] generates the correct mark chains", ver), func() { - masqBit := 5 - ch := genSetMarkChain(masqBit) - Expect(ch).To(Equal(chain{ - table: "nat", - name: "CNI-HOSTPORT-SETMARK", - rules: [][]string{{ - "-m", "comment", - "--comment", "CNI portfwd masquerade mark", - "-j", "MARK", - "--set-xmark", "0x20/0x20", - }}, - })) - - ch = genMarkMasqChain(masqBit) - Expect(ch).To(Equal(chain{ - table: "nat", - name: "CNI-HOSTPORT-MASQ", - entryChains: []string{"POSTROUTING"}, - entryRules: [][]string{{ - "-m", "comment", - "--comment", "CNI portfwd requiring masquerade", - }}, - rules: [][]string{{ - "-m", "mark", - "--mark", "0x20/0x20", - "-j", "MASQUERADE", - }}, - prependEntry: true, - })) - }) - }) - }) } }) diff --git a/plugins/meta/portmap/utils.go b/plugins/meta/portmap/utils.go index e6709089..92270557 100644 --- a/plugins/meta/portmap/utils.go +++ b/plugins/meta/portmap/utils.go @@ -21,6 +21,8 @@ import ( "strings" "github.com/vishvananda/netlink" + + "github.com/containernetworking/plugins/pkg/utils/sysctl" ) // fmtIpPort correctly formats ip:port literals for iptables and ip6tables - @@ -52,6 +54,14 @@ func getRoutableHostIF(containerIP net.IP) string { return "" } +// enableLocalnetRouting tells the kernel not to treat 127/8 as a martian, +// so that connections with a source ip of 127/8 can cross a routing boundary. +func enableLocalnetRouting(ifName string) error { + routeLocalnetPath := "net/ipv4/conf/" + ifName + "/route_localnet" + _, err := sysctl.Sysctl(routeLocalnetPath, "1") + return err +} + // groupByProto groups port numbers by protocol func groupByProto(entries []PortMapEntry) map[string][]int { if len(entries) == 0 {