From 9ce99d3f07f7a14c3db95918757db3d2605c12e7 Mon Sep 17 00:00:00 2001 From: David Verbeiren Date: Fri, 4 Sep 2020 11:11:48 +0200 Subject: [PATCH] flannel: allow input ipam parameters as basis for delegate This change allows providing an 'ipam' section as part of the input network configuration for flannel. It is then used as basis to construct the ipam parameters provided to the delegate. All parameters from the input ipam are preserved except: * 'subnet' which is set to the flannel host subnet * 'routes' which is complemented by a route to the flannel network. One use case of this feature is to allow adding back the routes to the cluster services and/or to the hosts (HostPort) when using isDefaultGateway=false. In that case, the bridge plugin does not install a default route and, as a result, only pod-to-pod connectivity would be available. Example: { "name": "cbr0", "cniVersion": "0.3.1", "type": "flannel", "ipam": { "routes": [ { "dst": "192.168.242.0/24" }, { "dst": "10.96.0.0/12" } ], "unknown-param": "value" }, "delegate": { "hairpinMode": true, "isDefaultGateway": false } ... } This results in the following 'ipam' being provided to the delegate: { "routes" : [ { "dst": "192.168.242.0/24" }, { "dst": "10.96.0.0/12" }, { "dst" : "10.1.0.0/16" } ], "subnet" : "10.1.17.0/24", "type" : "host-local" "unknown-param": "value" } where "10.1.0.0/16" is the flannel network and "10.1.17.0/24" is the host flannel subnet. Note that this also allows setting a different ipam 'type' than "host-local". Signed-off-by: David Verbeiren --- plugins/meta/flannel/README.md | 7 +- plugins/meta/flannel/flannel.go | 14 ++++ plugins/meta/flannel/flannel_linux.go | 38 +++++++--- plugins/meta/flannel/flannel_linux_test.go | 88 +++++++++++++++++++++- 4 files changed, 134 insertions(+), 13 deletions(-) diff --git a/plugins/meta/flannel/README.md b/plugins/meta/flannel/README.md index 326c7d58..a74f9247 100644 --- a/plugins/meta/flannel/README.md +++ b/plugins/meta/flannel/README.md @@ -68,6 +68,8 @@ To use `ipvlan` instead of `bridge`, the following configuration can be specifie } ``` +The `ipam` part of the network configuration generated for the delegated plugin, can also be customized by adding a base `ipam` section to the input flannel network configuration. This `ipam` element is then updated with the flannel subnet, a route to the flannel network and, unless provided, an `ipam` `type` set to `host-local` before being provided to the delegated plugin. + ## Network configuration reference * `name` (string, required): the name of the network @@ -75,11 +77,12 @@ To use `ipvlan` instead of `bridge`, the following configuration can be specifie * `subnetFile` (string, optional): full path to the subnet file written out by flanneld. Defaults to /run/flannel/subnet.env * `dataDir` (string, optional): path to directory where plugin will store generated network configuration files. Defaults to `/var/lib/cni/flannel` * `delegate` (dictionary, optional): specifies configuration options for the delegated plugin. +* `ipam` (dictionary, optional, Linux only): when specified, is used as basis to construct the `ipam` section of the delegated plugin configuration. flannel plugin will always set the following fields in the delegated plugin configuration: * `name`: value of its "name" field. -* `ipam`: "host-local" type will be used with "subnet" set to `$FLANNEL_SUBNET`. +* `ipam`: based on the received `ipam` section if present, with a `type` defaulting to `host-local`, a `subnet` set to `$FLANNEL_SUBNET` and (Linux only) a `routes` element assembled from the routes listed in the received `ipam` element and a route to the flannel network. Other fields present in the input `ipam` section will be transparently provided to the delegate. flannel plugin will set the following fields in the delegated plugin configuration if they are not present: * `ipMasq`: the inverse of `$FLANNEL_IPMASQ` @@ -87,6 +90,8 @@ flannel plugin will set the following fields in the delegated plugin configurati Additionally, for the bridge plugin, `isGateway` will be set to `true`, if not present. +One use case of the `ipam` configuration is to allow adding back the routes to the cluster services and/or to the hosts when using `isDefaultGateway=false`. In that case, the bridge plugin does not install a default route and, as a result, only pod-to-pod connectivity would be available. + ## Windows Support (Experimental) This plugin supports delegating to the windows CNI plugins (overlay.exe, l2bridge.exe) to work in conjunction with [Flannel on Windows](https://github.com/coreos/flannel/issues/833). Flannel sets up an [HNS Network](https://docs.microsoft.com/en-us/virtualization/windowscontainers/container-networking/architecture) in L2Bridge mode for host-gw and in Overlay mode for vxlan. diff --git a/plugins/meta/flannel/flannel.go b/plugins/meta/flannel/flannel.go index 9556797a..b3f4af9e 100644 --- a/plugins/meta/flannel/flannel.go +++ b/plugins/meta/flannel/flannel.go @@ -46,6 +46,8 @@ const ( type NetConf struct { types.NetConf + // IPAM field "replaces" that of types.NetConf which is incomplete + IPAM map[string]interface{} `json:"ipam,omitempty"` SubnetFile string `json:"subnetFile"` DataDir string `json:"dataDir"` Delegate map[string]interface{} `json:"delegate"` @@ -89,6 +91,18 @@ func loadFlannelNetConf(bytes []byte) (*NetConf, error) { return n, nil } +func getIPAMRoutes(n *NetConf) ([]types.Route, error) { + rtes := []types.Route{} + + if n.IPAM != nil && hasKey(n.IPAM, "routes") { + buf, _ := json.Marshal(n.IPAM["routes"]) + if err := json.Unmarshal(buf, &rtes); err != nil { + return rtes, fmt.Errorf("failed to parse ipam.routes: %w", err) + } + } + return rtes, nil +} + func loadFlannelSubnetEnv(fn string) (*subnetEnv, error) { f, err := os.Open(fn) if err != nil { diff --git a/plugins/meta/flannel/flannel_linux.go b/plugins/meta/flannel/flannel_linux.go index d0df309c..4f9906ee 100644 --- a/plugins/meta/flannel/flannel_linux.go +++ b/plugins/meta/flannel/flannel_linux.go @@ -22,12 +22,36 @@ import ( "context" "encoding/json" "fmt" + "os" + "github.com/containernetworking/cni/pkg/invoke" "github.com/containernetworking/cni/pkg/skel" "github.com/containernetworking/cni/pkg/types" - "os" ) +// Return IPAM section for Delegate using input IPAM if present and replacing +// or complementing as needed. +func getDelegateIPAM(n *NetConf, fenv *subnetEnv) (map[string]interface{}, error) { + ipam := n.IPAM + if ipam == nil { + ipam = map[string]interface{}{} + } + + if !hasKey(ipam, "type") { + ipam["type"] = "host-local" + } + ipam["subnet"] = fenv.sn.String() + + rtes, err := getIPAMRoutes(n) + if err != nil { + return nil, fmt.Errorf("failed to read IPAM routes: %w", err) + } + rtes = append(rtes, types.Route{Dst: *fenv.nw}) + ipam["routes"] = rtes + + return ipam, nil +} + func doCmdAdd(args *skel.CmdArgs, n *NetConf, fenv *subnetEnv) error { n.Delegate["name"] = n.Name @@ -55,15 +79,11 @@ func doCmdAdd(args *skel.CmdArgs, n *NetConf, fenv *subnetEnv) error { n.Delegate["cniVersion"] = n.CNIVersion } - n.Delegate["ipam"] = map[string]interface{}{ - "type": "host-local", - "subnet": fenv.sn.String(), - "routes": []types.Route{ - { - Dst: *fenv.nw, - }, - }, + ipam, err := getDelegateIPAM(n, fenv) + if err != nil { + return fmt.Errorf("failed to assemble Delegate IPAM: %w", err) } + n.Delegate["ipam"] = ipam return delegateAdd(args.ContainerID, n.DataDir, n.Delegate) } diff --git a/plugins/meta/flannel/flannel_linux_test.go b/plugins/meta/flannel/flannel_linux_test.go index 81125a28..83abdefd 100644 --- a/plugins/meta/flannel/flannel_linux_test.go +++ b/plugins/meta/flannel/flannel_linux_test.go @@ -14,6 +14,7 @@ package main import ( + "encoding/json" "fmt" "io/ioutil" "os" @@ -50,9 +51,25 @@ var _ = Describe("Flannel", func() { "name": "cni-flannel", "type": "flannel", "subnetFile": "%s", - "dataDir": "%s" + "dataDir": "%s"%s }` + const inputIPAMTemplate = ` + "unknown-param": "unknown-value", + "routes": [%s]%s` + + const inputIPAMType = "my-ipam" + + const inputIPAMNoTypeTemplate = ` +{ + "unknown-param": "unknown-value", + "routes": [%s]%s +}` + + const inputIPAMRoutes = ` + { "dst": "10.96.0.0/12" }, + { "dst": "192.168.244.0/24", "gw": "10.1.17.20" }` + const flannelSubnetEnv = ` FLANNEL_NETWORK=10.1.0.0/16 FLANNEL_SUBNET=10.1.17.1/24 @@ -68,6 +85,26 @@ FLANNEL_IPMASQ=true return file.Name() } + var makeInputIPAM = func(ipamType, routes, extra string) string { + c := "{\n" + if len(ipamType) > 0 { + c += fmt.Sprintf(" \"type\": \"%s\",", ipamType) + } + c += fmt.Sprintf(inputIPAMTemplate, routes, extra) + c += "\n}" + + return c + } + + var makeInput = func(inputIPAM string) string { + ipamPart := "" + if len(inputIPAM) > 0 { + ipamPart = ",\n \"ipam\":\n" + inputIPAM + } + + return fmt.Sprintf(inputTemplate, subnetFile, dataDir, ipamPart) + } + BeforeEach(func() { var err error // flannel subnet.env @@ -76,7 +113,7 @@ FLANNEL_IPMASQ=true // flannel state dir dataDir, err = ioutil.TempDir("", "dataDir") Expect(err).NotTo(HaveOccurred()) - input = fmt.Sprintf(inputTemplate, subnetFile, dataDir) + input = makeInput("") }) AfterEach(func() { @@ -108,7 +145,7 @@ FLANNEL_IPMASQ=true }) Expect(err).NotTo(HaveOccurred()) - By("check that plugin writes to net config to dataDir") + By("check that plugin writes the net config to dataDir") path := fmt.Sprintf("%s/%s", dataDir, "some-container-id") Expect(path).Should(BeAnExistingFile()) @@ -213,4 +250,49 @@ FLANNEL_IPMASQ=true }) }) }) + + Describe("getDelegateIPAM", func() { + Context("when input IPAM is provided", func() { + BeforeEach(func() { + inputIPAM := makeInputIPAM(inputIPAMType, inputIPAMRoutes, "") + input = makeInput(inputIPAM) + }) + It("configures Delegate IPAM accordingly", func() { + conf, err := loadFlannelNetConf([]byte(input)) + Expect(err).ShouldNot(HaveOccurred()) + fenv, err := loadFlannelSubnetEnv(subnetFile) + Expect(err).ShouldNot(HaveOccurred()) + + ipam, err := getDelegateIPAM(conf, fenv) + Expect(err).ShouldNot(HaveOccurred()) + + podsRoute := "{ \"dst\": \"10.1.0.0/16\" }\n" + subnet := "\"subnet\": \"10.1.17.0/24\"" + expected := makeInputIPAM(inputIPAMType, inputIPAMRoutes+",\n"+podsRoute, ",\n"+subnet) + buf, _ := json.Marshal(ipam) + Expect(buf).Should(MatchJSON(expected)) + }) + }) + + Context("when input IPAM is provided without 'type'", func() { + BeforeEach(func() { + inputIPAM := makeInputIPAM("", inputIPAMRoutes, "") + input = makeInput(inputIPAM) + }) + It("configures Delegate IPAM with 'host-local' ipam", func() { + conf, err := loadFlannelNetConf([]byte(input)) + Expect(err).ShouldNot(HaveOccurred()) + fenv, err := loadFlannelSubnetEnv(subnetFile) + Expect(err).ShouldNot(HaveOccurred()) + ipam, err := getDelegateIPAM(conf, fenv) + Expect(err).ShouldNot(HaveOccurred()) + + podsRoute := "{ \"dst\": \"10.1.0.0/16\" }\n" + subnet := "\"subnet\": \"10.1.17.0/24\"" + expected := makeInputIPAM("host-local", inputIPAMRoutes+",\n"+podsRoute, ",\n"+subnet) + buf, _ := json.Marshal(ipam) + Expect(buf).Should(MatchJSON(expected)) + }) + }) + }) })