Merge pull request #532 from dverbeir/flannel_ipam_routes

flannel: allow input ipam parameters as basis for delegate

plugins/meta/flannel/README.md

@@ -68,6 +68,8 @@ To use `ipvlan` instead of `bridge`, the following configuration can be specifie
 }
 ```
 
+The `ipam` part of the network configuration generated for the delegated plugin, can also be customized by adding a base `ipam` section to the input flannel network configuration. This `ipam` element is then updated with the flannel subnet, a route to the flannel network and, unless provided, an `ipam` `type` set to `host-local` before being provided to the delegated plugin.
+
 ## Network configuration reference
 
 * `name` (string, required): the name of the network
@@ -75,11 +77,12 @@ To use `ipvlan` instead of `bridge`, the following configuration can be specifie
 * `subnetFile` (string, optional): full path to the subnet file written out by flanneld. Defaults to /run/flannel/subnet.env
 * `dataDir` (string, optional): path to directory where plugin will store generated network configuration files. Defaults to `/var/lib/cni/flannel`
 * `delegate` (dictionary, optional): specifies configuration options for the delegated plugin.
+* `ipam` (dictionary, optional, Linux only): when specified, is used as basis to construct the `ipam` section of the delegated plugin configuration.
 
 flannel plugin will always set the following fields in the delegated plugin configuration:
 
 * `name`: value of its "name" field.
-* `ipam`: "host-local" type will be used with "subnet" set to `$FLANNEL_SUBNET`.
+* `ipam`: based on the received `ipam` section if present, with a `type` defaulting to `host-local`, a `subnet` set to `$FLANNEL_SUBNET` and (Linux only) a `routes` element assembled from the routes listed in the received `ipam` element and a route to the flannel network. Other fields present in the input `ipam` section will be transparently provided to the delegate.
 
 flannel plugin will set the following fields in the delegated plugin configuration if they are not present:
 * `ipMasq`: the inverse of `$FLANNEL_IPMASQ`
@@ -87,6 +90,8 @@ flannel plugin will set the following fields in the delegated plugin configurati
 
 Additionally, for the bridge plugin, `isGateway` will be set to `true`, if not present.
 
+One use case of the `ipam` configuration is to allow adding back the routes to the cluster services and/or to the hosts when using `isDefaultGateway=false`. In that case, the bridge plugin does not install a default route and, as a result, only pod-to-pod connectivity would be available.
+
 ## Windows Support (Experimental)
 This plugin supports delegating to the windows CNI plugins (overlay.exe, l2bridge.exe) to work in conjunction with [Flannel on Windows](https://github.com/coreos/flannel/issues/833). 
 Flannel sets up an [HNS Network](https://docs.microsoft.com/en-us/virtualization/windowscontainers/container-networking/architecture) in L2Bridge mode for host-gw and in Overlay mode for vxlan. 

plugins/meta/flannel/flannel.go

@@ -46,6 +46,8 @@ const (
 type NetConf struct {
 	types.NetConf
 
+	// IPAM field "replaces" that of types.NetConf which is incomplete
+	IPAM          map[string]interface{} `json:"ipam,omitempty"`
 	SubnetFile    string                 `json:"subnetFile"`
 	DataDir       string                 `json:"dataDir"`
 	Delegate      map[string]interface{} `json:"delegate"`
@@ -89,6 +91,18 @@ func loadFlannelNetConf(bytes []byte) (*NetConf, error) {
 	return n, nil
 }
 
+func getIPAMRoutes(n *NetConf) ([]types.Route, error) {
+	rtes := []types.Route{}
+
+	if n.IPAM != nil && hasKey(n.IPAM, "routes") {
+		buf, _ := json.Marshal(n.IPAM["routes"])
+		if err := json.Unmarshal(buf, &rtes); err != nil {
+			return rtes, fmt.Errorf("failed to parse ipam.routes: %w", err)
+		}
+	}
+	return rtes, nil
+}
+
 func loadFlannelSubnetEnv(fn string) (*subnetEnv, error) {
 	f, err := os.Open(fn)
 	if err != nil {

plugins/meta/flannel/flannel_linux.go

@@ -22,12 +22,36 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"os"
+
 	"github.com/containernetworking/cni/pkg/invoke"
 	"github.com/containernetworking/cni/pkg/skel"
 	"github.com/containernetworking/cni/pkg/types"
-	"os"
 )
 
+// Return IPAM section for Delegate using input IPAM if present and replacing
+// or complementing as needed.
+func getDelegateIPAM(n *NetConf, fenv *subnetEnv) (map[string]interface{}, error) {
+	ipam := n.IPAM
+	if ipam == nil {
+		ipam = map[string]interface{}{}
+	}
+
+	if !hasKey(ipam, "type") {
+		ipam["type"] = "host-local"
+	}
+	ipam["subnet"] = fenv.sn.String()
+
+	rtes, err := getIPAMRoutes(n)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read IPAM routes: %w", err)
+	}
+	rtes = append(rtes, types.Route{Dst: *fenv.nw})
+	ipam["routes"] = rtes
+
+	return ipam, nil
+}
+
 func doCmdAdd(args *skel.CmdArgs, n *NetConf, fenv *subnetEnv) error {
 	n.Delegate["name"] = n.Name
 
@@ -55,15 +79,11 @@ func doCmdAdd(args *skel.CmdArgs, n *NetConf, fenv *subnetEnv) error {
 		n.Delegate["cniVersion"] = n.CNIVersion
 	}
 
-	n.Delegate["ipam"] = map[string]interface{}{
+	ipam, err := getDelegateIPAM(n, fenv)
-		"type":   "host-local",
+	if err != nil {
-		"subnet": fenv.sn.String(),
+		return fmt.Errorf("failed to assemble Delegate IPAM: %w", err)
-		"routes": []types.Route{
-			{
-				Dst: *fenv.nw,
-			},
-		},
 	}
+	n.Delegate["ipam"] = ipam
 
 	return delegateAdd(args.ContainerID, n.DataDir, n.Delegate)
 }

plugins/meta/flannel/flannel_linux_test.go

@@ -14,6 +14,7 @@
 package main
 
 import (
+	"encoding/json"
 	"fmt"
 	"io/ioutil"
 	"os"
@@ -50,9 +51,25 @@ var _ = Describe("Flannel", func() {
   "name": "cni-flannel",
   "type": "flannel",
 	"subnetFile": "%s",
-	"dataDir": "%s"
+	"dataDir": "%s"%s
 }`
 
+	const inputIPAMTemplate = `
+    "unknown-param": "unknown-value",
+    "routes": [%s]%s`
+
+	const inputIPAMType = "my-ipam"
+
+	const inputIPAMNoTypeTemplate = `
+{
+    "unknown-param": "unknown-value",
+    "routes": [%s]%s
+}`
+
+	const inputIPAMRoutes = `
+      { "dst": "10.96.0.0/12" },
+      { "dst": "192.168.244.0/24", "gw": "10.1.17.20" }`
+
 	const flannelSubnetEnv = `
 FLANNEL_NETWORK=10.1.0.0/16
 FLANNEL_SUBNET=10.1.17.1/24
@@ -68,6 +85,26 @@ FLANNEL_IPMASQ=true
 		return file.Name()
 	}
 
+	var makeInputIPAM = func(ipamType, routes, extra string) string {
+		c := "{\n"
+		if len(ipamType) > 0 {
+			c += fmt.Sprintf("    \"type\": \"%s\",", ipamType)
+		}
+		c += fmt.Sprintf(inputIPAMTemplate, routes, extra)
+		c += "\n}"
+
+		return c
+	}
+
+	var makeInput = func(inputIPAM string) string {
+		ipamPart := ""
+		if len(inputIPAM) > 0 {
+			ipamPart = ",\n  \"ipam\":\n" + inputIPAM
+		}
+
+		return fmt.Sprintf(inputTemplate, subnetFile, dataDir, ipamPart)
+	}
+
 	BeforeEach(func() {
 		var err error
 		// flannel subnet.env
@@ -76,7 +113,7 @@ FLANNEL_IPMASQ=true
 		// flannel state dir
 		dataDir, err = ioutil.TempDir("", "dataDir")
 		Expect(err).NotTo(HaveOccurred())
-		input = fmt.Sprintf(inputTemplate, subnetFile, dataDir)
+		input = makeInput("")
 	})
 
 	AfterEach(func() {
@@ -108,7 +145,7 @@ FLANNEL_IPMASQ=true
 				})
 				Expect(err).NotTo(HaveOccurred())
 
-				By("check that plugin writes to net config to dataDir")
+				By("check that plugin writes the net config to dataDir")
 				path := fmt.Sprintf("%s/%s", dataDir, "some-container-id")
 				Expect(path).Should(BeAnExistingFile())
 
@@ -213,4 +250,49 @@ FLANNEL_IPMASQ=true
 			})
 		})
 	})
+
+	Describe("getDelegateIPAM", func() {
+		Context("when input IPAM is provided", func() {
+			BeforeEach(func() {
+				inputIPAM := makeInputIPAM(inputIPAMType, inputIPAMRoutes, "")
+				input = makeInput(inputIPAM)
+			})
+			It("configures Delegate IPAM accordingly", func() {
+				conf, err := loadFlannelNetConf([]byte(input))
+				Expect(err).ShouldNot(HaveOccurred())
+				fenv, err := loadFlannelSubnetEnv(subnetFile)
+				Expect(err).ShouldNot(HaveOccurred())
+
+				ipam, err := getDelegateIPAM(conf, fenv)
+				Expect(err).ShouldNot(HaveOccurred())
+
+				podsRoute := "{ \"dst\": \"10.1.0.0/16\" }\n"
+				subnet := "\"subnet\": \"10.1.17.0/24\""
+				expected := makeInputIPAM(inputIPAMType, inputIPAMRoutes+",\n"+podsRoute, ",\n"+subnet)
+				buf, _ := json.Marshal(ipam)
+				Expect(buf).Should(MatchJSON(expected))
+			})
+		})
+
+		Context("when input IPAM is provided without 'type'", func() {
+			BeforeEach(func() {
+				inputIPAM := makeInputIPAM("", inputIPAMRoutes, "")
+				input = makeInput(inputIPAM)
+			})
+			It("configures Delegate IPAM with 'host-local' ipam", func() {
+				conf, err := loadFlannelNetConf([]byte(input))
+				Expect(err).ShouldNot(HaveOccurred())
+				fenv, err := loadFlannelSubnetEnv(subnetFile)
+				Expect(err).ShouldNot(HaveOccurred())
+				ipam, err := getDelegateIPAM(conf, fenv)
+				Expect(err).ShouldNot(HaveOccurred())
+
+				podsRoute := "{ \"dst\": \"10.1.0.0/16\" }\n"
+				subnet := "\"subnet\": \"10.1.17.0/24\""
+				expected := makeInputIPAM("host-local", inputIPAMRoutes+",\n"+podsRoute, ",\n"+subnet)
+				buf, _ := json.Marshal(ipam)
+				Expect(buf).Should(MatchJSON(expected))
+			})
+		})
+	})
 })