Merge pull request #322 from cf-container-networking/flannel-cni-stateDir

Allow flannel CNI plugin stateDir to be configurable

Documentation/flannel.md

@@ -73,6 +73,7 @@ To use `ipvlan` instead of `bridge`, the following configuration can be specifie
 * `name` (string, required): the name of the network
 * `type` (string, required): "flannel"
 * `subnetFile` (string, optional): full path to the subnet file written out by flanneld. Defaults to /run/flannel/subnet.env
+* `dataDir` (string, optional): path to directory where plugin will store generated network configuration files. Defaults to `/var/lib/cni/flannel`
 * `delegate` (dictionary, optional): specifies configuration options for the delegated plugin.
 
 flannel plugin will always set the following fields in the delegated plugin configuration:

plugins/meta/flannel/flannel.go

@@ -37,12 +37,13 @@ import (
 
 const (
 	defaultSubnetFile = "/run/flannel/subnet.env"
-	stateDir          = "/var/lib/cni/flannel"
+	defaultDataDir    = "/var/lib/cni/flannel"
 )
 
 type NetConf struct {
 	types.NetConf
 	SubnetFile string                 `json:"subnetFile"`
+	DataDir    string                 `json:"dataDir"`
 	Delegate   map[string]interface{} `json:"delegate"`
 }
 
@@ -74,6 +75,7 @@ func (se *subnetEnv) missing() string {
 func loadFlannelNetConf(bytes []byte) (*NetConf, error) {
 	n := &NetConf{
 		SubnetFile: defaultSubnetFile,
+		DataDir:    defaultDataDir,
 	}
 	if err := json.Unmarshal(bytes, n); err != nil {
 		return nil, fmt.Errorf("failed to load netconf: %v", err)
@@ -130,29 +132,29 @@ func loadFlannelSubnetEnv(fn string) (*subnetEnv, error) {
 	return se, nil
 }
 
-func saveScratchNetConf(containerID string, netconf []byte) error {
+func saveScratchNetConf(containerID, dataDir string, netconf []byte) error {
-	if err := os.MkdirAll(stateDir, 0700); err != nil {
+	if err := os.MkdirAll(dataDir, 0700); err != nil {
 		return err
 	}
-	path := filepath.Join(stateDir, containerID)
+	path := filepath.Join(dataDir, containerID)
 	return ioutil.WriteFile(path, netconf, 0600)
 }
 
-func consumeScratchNetConf(containerID string) ([]byte, error) {
+func consumeScratchNetConf(containerID, dataDir string) ([]byte, error) {
-	path := filepath.Join(stateDir, containerID)
+	path := filepath.Join(dataDir, containerID)
 	defer os.Remove(path)
 
 	return ioutil.ReadFile(path)
 }
 
-func delegateAdd(cid string, netconf map[string]interface{}) error {
+func delegateAdd(cid, dataDir string, netconf map[string]interface{}) error {
 	netconfBytes, err := json.Marshal(netconf)
 	if err != nil {
 		return fmt.Errorf("error serializing delegate netconf: %v", err)
 	}
 
 	// save the rendered netconf for cmdDel
-	if err = saveScratchNetConf(cid, netconfBytes); err != nil {
+	if err = saveScratchNetConf(cid, dataDir, netconfBytes); err != nil {
 		return err
 	}
 
@@ -232,11 +234,16 @@ func cmdAdd(args *skel.CmdArgs) error {
 		},
 	}
 
-	return delegateAdd(args.ContainerID, n.Delegate)
+	return delegateAdd(args.ContainerID, n.DataDir, n.Delegate)
 }
 
 func cmdDel(args *skel.CmdArgs) error {
-	netconfBytes, err := consumeScratchNetConf(args.ContainerID)
+	nc, err := loadFlannelNetConf(args.StdinData)
+	if err != nil {
+		return err
+	}
+
+	netconfBytes, err := consumeScratchNetConf(args.ContainerID, nc.DataDir)
 	if err != nil {
 		return err
 	}

plugins/meta/flannel/flannel_suite_test.go

@@ -0,0 +1,26 @@
+// Copyright 2015 CNI authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package main
+
+import (
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"testing"
+)
+
+func TestFlannel(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Flannel Suite")
+}

plugins/meta/flannel/flannel_test.go

@@ -0,0 +1,203 @@
+// Copyright 2015 CNI authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+
+	"github.com/containernetworking/cni/pkg/ns"
+	"github.com/containernetworking/cni/pkg/skel"
+	"github.com/containernetworking/cni/pkg/testutils"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Flannel", func() {
+	var (
+		originalNS ns.NetNS
+		input      string
+		subnetFile string
+		dataDir    string
+	)
+
+	BeforeEach(func() {
+		var err error
+		originalNS, err = ns.NewNS()
+		Expect(err).NotTo(HaveOccurred())
+	})
+
+	AfterEach(func() {
+		Expect(originalNS.Close()).To(Succeed())
+	})
+
+	const inputTemplate = `
+{
+  "name": "cni-flannel",
+  "type": "flannel",
+	"subnetFile": "%s",
+	"dataDir": "%s"
+}`
+
+	const flannelSubnetEnv = `
+FLANNEL_NETWORK=10.1.0.0/16
+FLANNEL_SUBNET=10.1.17.1/24
+FLANNEL_MTU=1472
+FLANNEL_IPMASQ=true
+`
+
+	var writeSubnetEnv = func(contents string) string {
+		file, err := ioutil.TempFile("", "subnet.env")
+		Expect(err).NotTo(HaveOccurred())
+		_, err = file.WriteString(contents)
+		Expect(err).NotTo(HaveOccurred())
+		return file.Name()
+	}
+
+	BeforeEach(func() {
+		var err error
+		// flannel subnet.env
+		subnetFile = writeSubnetEnv(flannelSubnetEnv)
+
+		// flannel state dir
+		dataDir, err = ioutil.TempDir("", "dataDir")
+		Expect(err).NotTo(HaveOccurred())
+		input = fmt.Sprintf(inputTemplate, subnetFile, dataDir)
+	})
+
+	AfterEach(func() {
+		os.Remove(subnetFile)
+		os.Remove(dataDir)
+	})
+
+	Describe("CNI lifecycle", func() {
+		It("uses dataDir for storing network configuration", func() {
+			const IFNAME = "eth0"
+
+			targetNs, err := ns.NewNS()
+			Expect(err).NotTo(HaveOccurred())
+			defer targetNs.Close()
+
+			args := &skel.CmdArgs{
+				ContainerID: "some-container-id",
+				Netns:       targetNs.Path(),
+				IfName:      IFNAME,
+				StdinData:   []byte(input),
+			}
+
+			err = originalNS.Do(func(ns.NetNS) error {
+				defer GinkgoRecover()
+
+				By("calling ADD")
+				_, err := testutils.CmdAddWithResult(targetNs.Path(), IFNAME, func() error {
+					return cmdAdd(args)
+				})
+				Expect(err).NotTo(HaveOccurred())
+
+				By("check that plugin writes to net config to dataDir")
+				path := fmt.Sprintf("%s/%s", dataDir, "some-container-id")
+				Expect(path).Should(BeAnExistingFile())
+
+				netConfBytes, err := ioutil.ReadFile(path)
+				Expect(err).NotTo(HaveOccurred())
+				expected := `{
+   "name" : "cni-flannel",
+   "type" : "bridge",
+   "ipam" : {
+      "type" : "host-local",
+      "subnet" : "10.1.17.0/24",
+      "routes" : [
+         {
+            "dst" : "10.1.0.0/16"
+         }
+      ]
+   },
+   "mtu" : 1472,
+   "ipMasq" : false,
+   "isGateway": true
+}
+`
+				Expect(netConfBytes).Should(MatchJSON(expected))
+
+				By("calling DEL")
+				err = testutils.CmdDelWithResult(targetNs.Path(), IFNAME, func() error {
+					return cmdDel(args)
+				})
+				Expect(err).NotTo(HaveOccurred())
+
+				By("check that plugin removes net config from state dir")
+				Expect(path).ShouldNot(BeAnExistingFile())
+				return nil
+			})
+			Expect(err).NotTo(HaveOccurred())
+		})
+	})
+
+	Describe("loadFlannelNetConf", func() {
+		Context("when subnetFile and dataDir are specified", func() {
+			It("loads flannel network config", func() {
+				conf, err := loadFlannelNetConf([]byte(input))
+				Expect(err).ShouldNot(HaveOccurred())
+				Expect(conf.Name).To(Equal("cni-flannel"))
+				Expect(conf.Type).To(Equal("flannel"))
+				Expect(conf.SubnetFile).To(Equal(subnetFile))
+				Expect(conf.DataDir).To(Equal(dataDir))
+			})
+		})
+
+		Context("when defaulting subnetFile and dataDir", func() {
+			BeforeEach(func() {
+				input = `{
+"name": "cni-flannel",
+"type": "flannel"
+}`
+			})
+
+			It("loads flannel network config with defaults", func() {
+				conf, err := loadFlannelNetConf([]byte(input))
+				Expect(err).ShouldNot(HaveOccurred())
+				Expect(conf.Name).To(Equal("cni-flannel"))
+				Expect(conf.Type).To(Equal("flannel"))
+				Expect(conf.SubnetFile).To(Equal(defaultSubnetFile))
+				Expect(conf.DataDir).To(Equal(defaultDataDir))
+			})
+		})
+
+		Describe("loadFlannelSubnetEnv", func() {
+			Context("when flannel subnet env is valid", func() {
+				It("loads flannel subnet config", func() {
+					conf, err := loadFlannelSubnetEnv(subnetFile)
+					Expect(err).ShouldNot(HaveOccurred())
+					Expect(conf.nw.String()).To(Equal("10.1.0.0/16"))
+					Expect(conf.sn.String()).To(Equal("10.1.17.0/24"))
+					var mtu uint = 1472
+					Expect(*conf.mtu).To(Equal(mtu))
+					Expect(*conf.ipmasq).To(BeTrue())
+				})
+			})
+
+			Context("when flannel subnet env is invalid", func() {
+				BeforeEach(func() {
+					subnetFile = writeSubnetEnv("foo=bar")
+				})
+				It("returns an error", func() {
+					_, err := loadFlannelSubnetEnv(subnetFile)
+					Expect(err).To(MatchError(ContainSubstring("missing FLANNEL_NETWORK, FLANNEL_SUBNET, FLANNEL_MTU, FLANNEL_IPMASQ")))
+				})
+			})
+		})
+	})
+})

test

@@ -11,7 +11,7 @@ set -e
 
 source ./build
 
-TESTABLE="libcni plugins/ipam/dhcp plugins/ipam/host-local plugins/main/loopback pkg/invoke pkg/ns pkg/skel pkg/types pkg/utils plugins/main/ipvlan plugins/main/macvlan plugins/main/bridge plugins/main/ptp plugins/test/noop pkg/utils/hwaddr pkg/ip pkg/version pkg/version/testhelpers"
+TESTABLE="libcni plugins/ipam/dhcp plugins/ipam/host-local plugins/main/loopback pkg/invoke pkg/ns pkg/skel pkg/types pkg/utils plugins/main/ipvlan plugins/main/macvlan plugins/main/bridge plugins/main/ptp plugins/test/noop pkg/utils/hwaddr pkg/ip pkg/version pkg/version/testhelpers plugins/meta/flannel"
 FORMATTABLE="$TESTABLE pkg/testutils plugins/meta/flannel plugins/meta/tuning"
 
 # user has not provided PKG override