flannel: allow input ipam parameters as basis for delegate
This change allows providing an 'ipam' section as part of the
input network configuration for flannel. It is then used as
basis to construct the ipam parameters provided to the delegate.
All parameters from the input ipam are preserved except:
* 'subnet' which is set to the flannel host subnet
* 'routes' which is complemented by a route to the flannel
network.
One use case of this feature is to allow adding back the routes
to the cluster services and/or to the hosts (HostPort) when
using isDefaultGateway=false. In that case, the bridge plugin
does not install a default route and, as a result, only pod-to-pod
connectivity would be available.
Example:
{
"name": "cbr0",
"cniVersion": "0.3.1",
"type": "flannel",
"ipam": {
"routes": [
{
"dst": "192.168.242.0/24"
},
{
"dst": "10.96.0.0/12"
}
],
"unknown-param": "value"
},
"delegate": {
"hairpinMode": true,
"isDefaultGateway": false
}
...
}
This results in the following 'ipam' being provided to the delegate:
{
"routes" : [
{
"dst": "192.168.242.0/24"
},
{
"dst": "10.96.0.0/12"
},
{
"dst" : "10.1.0.0/16"
}
],
"subnet" : "10.1.17.0/24",
"type" : "host-local"
"unknown-param": "value"
}
where "10.1.0.0/16" is the flannel network and "10.1.17.0/24" is
the host flannel subnet.
Note that this also allows setting a different ipam 'type' than
"host-local".
Signed-off-by: David Verbeiren <david.verbeiren@tessares.net>
This commit is contained in:
David Verbeiren
parent
e78e6aa5b9
commit
9ce99d3f07
@ -68,6 +68,8 @@ To use `ipvlan` instead of `bridge`, the following configuration can be specifie
|
||||
}
|
||||
```
|
||||
|
||||
The `ipam` part of the network configuration generated for the delegated plugin, can also be customized by adding a base `ipam` section to the input flannel network configuration. This `ipam` element is then updated with the flannel subnet, a route to the flannel network and, unless provided, an `ipam` `type` set to `host-local` before being provided to the delegated plugin.
|
||||
|
||||
## Network configuration reference
|
||||
|
||||
* `name` (string, required): the name of the network
|
||||
@ -75,11 +77,12 @@ To use `ipvlan` instead of `bridge`, the following configuration can be specifie
|
||||
* `subnetFile` (string, optional): full path to the subnet file written out by flanneld. Defaults to /run/flannel/subnet.env
|
||||
* `dataDir` (string, optional): path to directory where plugin will store generated network configuration files. Defaults to `/var/lib/cni/flannel`
|
||||
* `delegate` (dictionary, optional): specifies configuration options for the delegated plugin.
|
||||
* `ipam` (dictionary, optional, Linux only): when specified, is used as basis to construct the `ipam` section of the delegated plugin configuration.
|
||||
|
||||
flannel plugin will always set the following fields in the delegated plugin configuration:
|
||||
|
||||
* `name`: value of its "name" field.
|
||||
* `ipam`: "host-local" type will be used with "subnet" set to `$FLANNEL_SUBNET`.
|
||||
* `ipam`: based on the received `ipam` section if present, with a `type` defaulting to `host-local`, a `subnet` set to `$FLANNEL_SUBNET` and (Linux only) a `routes` element assembled from the routes listed in the received `ipam` element and a route to the flannel network. Other fields present in the input `ipam` section will be transparently provided to the delegate.
|
||||
|
||||
flannel plugin will set the following fields in the delegated plugin configuration if they are not present:
|
||||
* `ipMasq`: the inverse of `$FLANNEL_IPMASQ`
|
||||
@ -87,6 +90,8 @@ flannel plugin will set the following fields in the delegated plugin configurati
|
||||
|
||||
Additionally, for the bridge plugin, `isGateway` will be set to `true`, if not present.
|
||||
|
||||
One use case of the `ipam` configuration is to allow adding back the routes to the cluster services and/or to the hosts when using `isDefaultGateway=false`. In that case, the bridge plugin does not install a default route and, as a result, only pod-to-pod connectivity would be available.
|
||||
|
||||
## Windows Support (Experimental)
|
||||
This plugin supports delegating to the windows CNI plugins (overlay.exe, l2bridge.exe) to work in conjunction with [Flannel on Windows](https://github.com/coreos/flannel/issues/833).
|
||||
Flannel sets up an [HNS Network](https://docs.microsoft.com/en-us/virtualization/windowscontainers/container-networking/architecture) in L2Bridge mode for host-gw and in Overlay mode for vxlan.
|
||||
|
||||
@ -46,6 +46,8 @@ const (
|
||||
type NetConf struct {
|
||||
types.NetConf
|
||||
|
||||
// IPAM field "replaces" that of types.NetConf which is incomplete
|
||||
IPAM map[string]interface{} `json:"ipam,omitempty"`
|
||||
SubnetFile string `json:"subnetFile"`
|
||||
DataDir string `json:"dataDir"`
|
||||
Delegate map[string]interface{} `json:"delegate"`
|
||||
@ -89,6 +91,18 @@ func loadFlannelNetConf(bytes []byte) (*NetConf, error) {
|
||||
return n, nil
|
||||
}
|
||||
|
||||
func getIPAMRoutes(n *NetConf) ([]types.Route, error) {
|
||||
rtes := []types.Route{}
|
||||
|
||||
if n.IPAM != nil && hasKey(n.IPAM, "routes") {
|
||||
buf, _ := json.Marshal(n.IPAM["routes"])
|
||||
if err := json.Unmarshal(buf, &rtes); err != nil {
|
||||
return rtes, fmt.Errorf("failed to parse ipam.routes: %w", err)
|
||||
}
|
||||
}
|
||||
return rtes, nil
|
||||
}
|
||||
|
||||
func loadFlannelSubnetEnv(fn string) (*subnetEnv, error) {
|
||||
f, err := os.Open(fn)
|
||||
if err != nil {
|
||||
|
||||
@ -22,12 +22,36 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"github.com/containernetworking/cni/pkg/invoke"
|
||||
"github.com/containernetworking/cni/pkg/skel"
|
||||
"github.com/containernetworking/cni/pkg/types"
|
||||
"os"
|
||||
)
|
||||
|
||||
// Return IPAM section for Delegate using input IPAM if present and replacing
|
||||
// or complementing as needed.
|
||||
func getDelegateIPAM(n *NetConf, fenv *subnetEnv) (map[string]interface{}, error) {
|
||||
ipam := n.IPAM
|
||||
if ipam == nil {
|
||||
ipam = map[string]interface{}{}
|
||||
}
|
||||
|
||||
if !hasKey(ipam, "type") {
|
||||
ipam["type"] = "host-local"
|
||||
}
|
||||
ipam["subnet"] = fenv.sn.String()
|
||||
|
||||
rtes, err := getIPAMRoutes(n)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read IPAM routes: %w", err)
|
||||
}
|
||||
rtes = append(rtes, types.Route{Dst: *fenv.nw})
|
||||
ipam["routes"] = rtes
|
||||
|
||||
return ipam, nil
|
||||
}
|
||||
|
||||
func doCmdAdd(args *skel.CmdArgs, n *NetConf, fenv *subnetEnv) error {
|
||||
n.Delegate["name"] = n.Name
|
||||
|
||||
@ -55,15 +79,11 @@ func doCmdAdd(args *skel.CmdArgs, n *NetConf, fenv *subnetEnv) error {
|
||||
n.Delegate["cniVersion"] = n.CNIVersion
|
||||
}
|
||||
|
||||
n.Delegate["ipam"] = map[string]interface{}{
|
||||
"type": "host-local",
|
||||
"subnet": fenv.sn.String(),
|
||||
"routes": []types.Route{
|
||||
{
|
||||
Dst: *fenv.nw,
|
||||
},
|
||||
},
|
||||
ipam, err := getDelegateIPAM(n, fenv)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to assemble Delegate IPAM: %w", err)
|
||||
}
|
||||
n.Delegate["ipam"] = ipam
|
||||
|
||||
return delegateAdd(args.ContainerID, n.DataDir, n.Delegate)
|
||||
}
|
||||
|
||||
@ -14,6 +14,7 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
@ -50,9 +51,25 @@ var _ = Describe("Flannel", func() {
|
||||
"name": "cni-flannel",
|
||||
"type": "flannel",
|
||||
"subnetFile": "%s",
|
||||
"dataDir": "%s"
|
||||
"dataDir": "%s"%s
|
||||
}`
|
||||
|
||||
const inputIPAMTemplate = `
|
||||
"unknown-param": "unknown-value",
|
||||
"routes": [%s]%s`
|
||||
|
||||
const inputIPAMType = "my-ipam"
|
||||
|
||||
const inputIPAMNoTypeTemplate = `
|
||||
{
|
||||
"unknown-param": "unknown-value",
|
||||
"routes": [%s]%s
|
||||
}`
|
||||
|
||||
const inputIPAMRoutes = `
|
||||
{ "dst": "10.96.0.0/12" },
|
||||
{ "dst": "192.168.244.0/24", "gw": "10.1.17.20" }`
|
||||
|
||||
const flannelSubnetEnv = `
|
||||
FLANNEL_NETWORK=10.1.0.0/16
|
||||
FLANNEL_SUBNET=10.1.17.1/24
|
||||
@ -68,6 +85,26 @@ FLANNEL_IPMASQ=true
|
||||
return file.Name()
|
||||
}
|
||||
|
||||
var makeInputIPAM = func(ipamType, routes, extra string) string {
|
||||
c := "{\n"
|
||||
if len(ipamType) > 0 {
|
||||
c += fmt.Sprintf(" \"type\": \"%s\",", ipamType)
|
||||
}
|
||||
c += fmt.Sprintf(inputIPAMTemplate, routes, extra)
|
||||
c += "\n}"
|
||||
|
||||
return c
|
||||
}
|
||||
|
||||
var makeInput = func(inputIPAM string) string {
|
||||
ipamPart := ""
|
||||
if len(inputIPAM) > 0 {
|
||||
ipamPart = ",\n \"ipam\":\n" + inputIPAM
|
||||
}
|
||||
|
||||
return fmt.Sprintf(inputTemplate, subnetFile, dataDir, ipamPart)
|
||||
}
|
||||
|
||||
BeforeEach(func() {
|
||||
var err error
|
||||
// flannel subnet.env
|
||||
@ -76,7 +113,7 @@ FLANNEL_IPMASQ=true
|
||||
// flannel state dir
|
||||
dataDir, err = ioutil.TempDir("", "dataDir")
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
input = fmt.Sprintf(inputTemplate, subnetFile, dataDir)
|
||||
input = makeInput("")
|
||||
})
|
||||
|
||||
AfterEach(func() {
|
||||
@ -108,7 +145,7 @@ FLANNEL_IPMASQ=true
|
||||
})
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
By("check that plugin writes to net config to dataDir")
|
||||
By("check that plugin writes the net config to dataDir")
|
||||
path := fmt.Sprintf("%s/%s", dataDir, "some-container-id")
|
||||
Expect(path).Should(BeAnExistingFile())
|
||||
|
||||
@ -213,4 +250,49 @@ FLANNEL_IPMASQ=true
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
Describe("getDelegateIPAM", func() {
|
||||
Context("when input IPAM is provided", func() {
|
||||
BeforeEach(func() {
|
||||
inputIPAM := makeInputIPAM(inputIPAMType, inputIPAMRoutes, "")
|
||||
input = makeInput(inputIPAM)
|
||||
})
|
||||
It("configures Delegate IPAM accordingly", func() {
|
||||
conf, err := loadFlannelNetConf([]byte(input))
|
||||
Expect(err).ShouldNot(HaveOccurred())
|
||||
fenv, err := loadFlannelSubnetEnv(subnetFile)
|
||||
Expect(err).ShouldNot(HaveOccurred())
|
||||
|
||||
ipam, err := getDelegateIPAM(conf, fenv)
|
||||
Expect(err).ShouldNot(HaveOccurred())
|
||||
|
||||
podsRoute := "{ \"dst\": \"10.1.0.0/16\" }\n"
|
||||
subnet := "\"subnet\": \"10.1.17.0/24\""
|
||||
expected := makeInputIPAM(inputIPAMType, inputIPAMRoutes+",\n"+podsRoute, ",\n"+subnet)
|
||||
buf, _ := json.Marshal(ipam)
|
||||
Expect(buf).Should(MatchJSON(expected))
|
||||
})
|
||||
})
|
||||
|
||||
Context("when input IPAM is provided without 'type'", func() {
|
||||
BeforeEach(func() {
|
||||
inputIPAM := makeInputIPAM("", inputIPAMRoutes, "")
|
||||
input = makeInput(inputIPAM)
|
||||
})
|
||||
It("configures Delegate IPAM with 'host-local' ipam", func() {
|
||||
conf, err := loadFlannelNetConf([]byte(input))
|
||||
Expect(err).ShouldNot(HaveOccurred())
|
||||
fenv, err := loadFlannelSubnetEnv(subnetFile)
|
||||
Expect(err).ShouldNot(HaveOccurred())
|
||||
ipam, err := getDelegateIPAM(conf, fenv)
|
||||
Expect(err).ShouldNot(HaveOccurred())
|
||||
|
||||
podsRoute := "{ \"dst\": \"10.1.0.0/16\" }\n"
|
||||
subnet := "\"subnet\": \"10.1.17.0/24\""
|
||||
expected := makeInputIPAM("host-local", inputIPAMRoutes+",\n"+podsRoute, ",\n"+subnet)
|
||||
buf, _ := json.Marshal(ipam)
|
||||
Expect(buf).Should(MatchJSON(expected))
|
||||
})
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user