diff --git a/plugins/meta/flannel/README.md b/plugins/meta/flannel/README.md index 326c7d58..a74f9247 100644 --- a/plugins/meta/flannel/README.md +++ b/plugins/meta/flannel/README.md @@ -68,6 +68,8 @@ To use `ipvlan` instead of `bridge`, the following configuration can be specifie } ``` +The `ipam` part of the network configuration generated for the delegated plugin, can also be customized by adding a base `ipam` section to the input flannel network configuration. This `ipam` element is then updated with the flannel subnet, a route to the flannel network and, unless provided, an `ipam` `type` set to `host-local` before being provided to the delegated plugin. + ## Network configuration reference * `name` (string, required): the name of the network @@ -75,11 +77,12 @@ To use `ipvlan` instead of `bridge`, the following configuration can be specifie * `subnetFile` (string, optional): full path to the subnet file written out by flanneld. Defaults to /run/flannel/subnet.env * `dataDir` (string, optional): path to directory where plugin will store generated network configuration files. Defaults to `/var/lib/cni/flannel` * `delegate` (dictionary, optional): specifies configuration options for the delegated plugin. +* `ipam` (dictionary, optional, Linux only): when specified, is used as basis to construct the `ipam` section of the delegated plugin configuration. flannel plugin will always set the following fields in the delegated plugin configuration: * `name`: value of its "name" field. -* `ipam`: "host-local" type will be used with "subnet" set to `$FLANNEL_SUBNET`. +* `ipam`: based on the received `ipam` section if present, with a `type` defaulting to `host-local`, a `subnet` set to `$FLANNEL_SUBNET` and (Linux only) a `routes` element assembled from the routes listed in the received `ipam` element and a route to the flannel network. Other fields present in the input `ipam` section will be transparently provided to the delegate. flannel plugin will set the following fields in the delegated plugin configuration if they are not present: * `ipMasq`: the inverse of `$FLANNEL_IPMASQ` @@ -87,6 +90,8 @@ flannel plugin will set the following fields in the delegated plugin configurati Additionally, for the bridge plugin, `isGateway` will be set to `true`, if not present. +One use case of the `ipam` configuration is to allow adding back the routes to the cluster services and/or to the hosts when using `isDefaultGateway=false`. In that case, the bridge plugin does not install a default route and, as a result, only pod-to-pod connectivity would be available. + ## Windows Support (Experimental) This plugin supports delegating to the windows CNI plugins (overlay.exe, l2bridge.exe) to work in conjunction with [Flannel on Windows](https://github.com/coreos/flannel/issues/833). Flannel sets up an [HNS Network](https://docs.microsoft.com/en-us/virtualization/windowscontainers/container-networking/architecture) in L2Bridge mode for host-gw and in Overlay mode for vxlan. diff --git a/plugins/meta/flannel/flannel.go b/plugins/meta/flannel/flannel.go index 9556797a..b3f4af9e 100644 --- a/plugins/meta/flannel/flannel.go +++ b/plugins/meta/flannel/flannel.go @@ -46,6 +46,8 @@ const ( type NetConf struct { types.NetConf + // IPAM field "replaces" that of types.NetConf which is incomplete + IPAM map[string]interface{} `json:"ipam,omitempty"` SubnetFile string `json:"subnetFile"` DataDir string `json:"dataDir"` Delegate map[string]interface{} `json:"delegate"` @@ -89,6 +91,18 @@ func loadFlannelNetConf(bytes []byte) (*NetConf, error) { return n, nil } +func getIPAMRoutes(n *NetConf) ([]types.Route, error) { + rtes := []types.Route{} + + if n.IPAM != nil && hasKey(n.IPAM, "routes") { + buf, _ := json.Marshal(n.IPAM["routes"]) + if err := json.Unmarshal(buf, &rtes); err != nil { + return rtes, fmt.Errorf("failed to parse ipam.routes: %w", err) + } + } + return rtes, nil +} + func loadFlannelSubnetEnv(fn string) (*subnetEnv, error) { f, err := os.Open(fn) if err != nil { diff --git a/plugins/meta/flannel/flannel_linux.go b/plugins/meta/flannel/flannel_linux.go index d0df309c..4f9906ee 100644 --- a/plugins/meta/flannel/flannel_linux.go +++ b/plugins/meta/flannel/flannel_linux.go @@ -22,12 +22,36 @@ import ( "context" "encoding/json" "fmt" + "os" + "github.com/containernetworking/cni/pkg/invoke" "github.com/containernetworking/cni/pkg/skel" "github.com/containernetworking/cni/pkg/types" - "os" ) +// Return IPAM section for Delegate using input IPAM if present and replacing +// or complementing as needed. +func getDelegateIPAM(n *NetConf, fenv *subnetEnv) (map[string]interface{}, error) { + ipam := n.IPAM + if ipam == nil { + ipam = map[string]interface{}{} + } + + if !hasKey(ipam, "type") { + ipam["type"] = "host-local" + } + ipam["subnet"] = fenv.sn.String() + + rtes, err := getIPAMRoutes(n) + if err != nil { + return nil, fmt.Errorf("failed to read IPAM routes: %w", err) + } + rtes = append(rtes, types.Route{Dst: *fenv.nw}) + ipam["routes"] = rtes + + return ipam, nil +} + func doCmdAdd(args *skel.CmdArgs, n *NetConf, fenv *subnetEnv) error { n.Delegate["name"] = n.Name @@ -55,15 +79,11 @@ func doCmdAdd(args *skel.CmdArgs, n *NetConf, fenv *subnetEnv) error { n.Delegate["cniVersion"] = n.CNIVersion } - n.Delegate["ipam"] = map[string]interface{}{ - "type": "host-local", - "subnet": fenv.sn.String(), - "routes": []types.Route{ - { - Dst: *fenv.nw, - }, - }, + ipam, err := getDelegateIPAM(n, fenv) + if err != nil { + return fmt.Errorf("failed to assemble Delegate IPAM: %w", err) } + n.Delegate["ipam"] = ipam return delegateAdd(args.ContainerID, n.DataDir, n.Delegate) } diff --git a/plugins/meta/flannel/flannel_linux_test.go b/plugins/meta/flannel/flannel_linux_test.go index 81125a28..83abdefd 100644 --- a/plugins/meta/flannel/flannel_linux_test.go +++ b/plugins/meta/flannel/flannel_linux_test.go @@ -14,6 +14,7 @@ package main import ( + "encoding/json" "fmt" "io/ioutil" "os" @@ -50,9 +51,25 @@ var _ = Describe("Flannel", func() { "name": "cni-flannel", "type": "flannel", "subnetFile": "%s", - "dataDir": "%s" + "dataDir": "%s"%s }` + const inputIPAMTemplate = ` + "unknown-param": "unknown-value", + "routes": [%s]%s` + + const inputIPAMType = "my-ipam" + + const inputIPAMNoTypeTemplate = ` +{ + "unknown-param": "unknown-value", + "routes": [%s]%s +}` + + const inputIPAMRoutes = ` + { "dst": "10.96.0.0/12" }, + { "dst": "192.168.244.0/24", "gw": "10.1.17.20" }` + const flannelSubnetEnv = ` FLANNEL_NETWORK=10.1.0.0/16 FLANNEL_SUBNET=10.1.17.1/24 @@ -68,6 +85,26 @@ FLANNEL_IPMASQ=true return file.Name() } + var makeInputIPAM = func(ipamType, routes, extra string) string { + c := "{\n" + if len(ipamType) > 0 { + c += fmt.Sprintf(" \"type\": \"%s\",", ipamType) + } + c += fmt.Sprintf(inputIPAMTemplate, routes, extra) + c += "\n}" + + return c + } + + var makeInput = func(inputIPAM string) string { + ipamPart := "" + if len(inputIPAM) > 0 { + ipamPart = ",\n \"ipam\":\n" + inputIPAM + } + + return fmt.Sprintf(inputTemplate, subnetFile, dataDir, ipamPart) + } + BeforeEach(func() { var err error // flannel subnet.env @@ -76,7 +113,7 @@ FLANNEL_IPMASQ=true // flannel state dir dataDir, err = ioutil.TempDir("", "dataDir") Expect(err).NotTo(HaveOccurred()) - input = fmt.Sprintf(inputTemplate, subnetFile, dataDir) + input = makeInput("") }) AfterEach(func() { @@ -108,7 +145,7 @@ FLANNEL_IPMASQ=true }) Expect(err).NotTo(HaveOccurred()) - By("check that plugin writes to net config to dataDir") + By("check that plugin writes the net config to dataDir") path := fmt.Sprintf("%s/%s", dataDir, "some-container-id") Expect(path).Should(BeAnExistingFile()) @@ -213,4 +250,49 @@ FLANNEL_IPMASQ=true }) }) }) + + Describe("getDelegateIPAM", func() { + Context("when input IPAM is provided", func() { + BeforeEach(func() { + inputIPAM := makeInputIPAM(inputIPAMType, inputIPAMRoutes, "") + input = makeInput(inputIPAM) + }) + It("configures Delegate IPAM accordingly", func() { + conf, err := loadFlannelNetConf([]byte(input)) + Expect(err).ShouldNot(HaveOccurred()) + fenv, err := loadFlannelSubnetEnv(subnetFile) + Expect(err).ShouldNot(HaveOccurred()) + + ipam, err := getDelegateIPAM(conf, fenv) + Expect(err).ShouldNot(HaveOccurred()) + + podsRoute := "{ \"dst\": \"10.1.0.0/16\" }\n" + subnet := "\"subnet\": \"10.1.17.0/24\"" + expected := makeInputIPAM(inputIPAMType, inputIPAMRoutes+",\n"+podsRoute, ",\n"+subnet) + buf, _ := json.Marshal(ipam) + Expect(buf).Should(MatchJSON(expected)) + }) + }) + + Context("when input IPAM is provided without 'type'", func() { + BeforeEach(func() { + inputIPAM := makeInputIPAM("", inputIPAMRoutes, "") + input = makeInput(inputIPAM) + }) + It("configures Delegate IPAM with 'host-local' ipam", func() { + conf, err := loadFlannelNetConf([]byte(input)) + Expect(err).ShouldNot(HaveOccurred()) + fenv, err := loadFlannelSubnetEnv(subnetFile) + Expect(err).ShouldNot(HaveOccurred()) + ipam, err := getDelegateIPAM(conf, fenv) + Expect(err).ShouldNot(HaveOccurred()) + + podsRoute := "{ \"dst\": \"10.1.0.0/16\" }\n" + subnet := "\"subnet\": \"10.1.17.0/24\"" + expected := makeInputIPAM("host-local", inputIPAMRoutes+",\n"+podsRoute, ",\n"+subnet) + buf, _ := json.Marshal(ipam) + Expect(buf).Should(MatchJSON(expected)) + }) + }) + }) })