ITQC/containernetworking-plugins
24
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 34 Wiki Activity

Revert "Merge pull request #921 from oOraph/dev/exclude_subnets_from_traffic_shapping2"

Browse Source 
This reverts commit ef076afac1af0b9a8446f72e3343666567bc04dc, reversing
changes made to 597408952e3e7247fb0deef26a3a935c405aa0cf.

Signed-off-by: h0nIg <h0nIg@users.noreply.github.com>
This commit is contained in:
h0nIg 2024-10-11 10:03:44 +00:00 committed by Casey Callendrello
parent 8ad0361964
commit d44bbf28af
9 changed files with 652 additions and 2928 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
integration/integration_linux_test.go
View File

@ -17,7 +17,6 @@ import (
"bytes"
"fmt"
"io"
"log"
"math/rand"
"net"
"os"
@ -61,13 +60,6 @@ var _ = Describe("Basic PTP using cnitool", func() {
netConfPath, err := filepath.Abs("./testdata")
Expect(err).NotTo(HaveOccurred())
// Flush ipam stores to avoid conflicts
err = os.RemoveAll("/tmp/chained-ptp-bandwidth-test")
Expect(err).NotTo(HaveOccurred())
err = os.RemoveAll("/tmp/basic-ptp-test")
Expect(err).NotTo(HaveOccurred())
env = TestEnv([]string{
"CNI_PATH=" + cniPath,
"NETCONFPATH=" + netConfPath,
@ -90,7 +82,6 @@ var _ = Describe("Basic PTP using cnitool", func() {
env.runInNS(hostNS, cnitoolBin, "add", netName, contNS.LongName())
addrOutput := env.runInNS(contNS, "ip", "addr")
Expect(addrOutput).To(ContainSubstring(expectedIPPrefix))
env.runInNS(hostNS, cnitoolBin, "del", netName, contNS.LongName())
@ -154,13 +145,9 @@ var _ = Describe("Basic PTP using cnitool", func() {
chainedBridgeBandwidthEnv.runInNS(hostNS, cnitoolBin, "del", "network-chain-test", contNS1.LongName())
basicBridgeEnv.runInNS(hostNS, cnitoolBin, "del", "network-chain-test", contNS2.LongName())
contNS1.Del()
contNS2.Del()
hostNS.Del()
})
It("limits traffic only on the restricted bandwidth veth device", func() {
Measure("limits traffic only on the restricted bandwidth veth device", func(b Benchmarker) {
ipRegexp := regexp.MustCompile(`10\.1[12]\.2\.\d{1,3}`)
By(fmt.Sprintf("adding %s to %s\n\n", "chained-bridge-bandwidth", contNS1.ShortName()))
@ -181,23 +168,21 @@ var _ = Describe("Basic PTP using cnitool", func() {
By(fmt.Sprintf("starting echo server in %s\n\n", contNS2.ShortName()))
basicBridgePort, basicBridgeSession = startEchoServerInNamespace(contNS2)
packetInBytes := 3000
packetInBytes := 20000 // The shaper needs to 'warm'. Send enough to cause it to throttle,
// balanced by run time.
By(fmt.Sprintf("sending tcp traffic to the chained, bridged, traffic shaped container on ip address '%s:%d'\n\n", chainedBridgeIP, chainedBridgeBandwidthPort))
start := time.Now()
makeTCPClientInNS(hostNS.ShortName(), chainedBridgeIP, chainedBridgeBandwidthPort, packetInBytes)
runtimeWithLimit := time.Since(start)
log.Printf("Runtime with qos limit %.2f seconds", runtimeWithLimit.Seconds())
runtimeWithLimit := b.Time("with chained bridge and bandwidth plugins", func() {
makeTCPClientInNS(hostNS.ShortName(), chainedBridgeIP, chainedBridgeBandwidthPort, packetInBytes)
})
By(fmt.Sprintf("sending tcp traffic to the basic bridged container on ip address '%s:%d'\n\n", basicBridgeIP, basicBridgePort))
start = time.Now()
makeTCPClientInNS(hostNS.ShortName(), basicBridgeIP, basicBridgePort, packetInBytes)
runtimeWithoutLimit := time.Since(start)
log.Printf("Runtime without qos limit %.2f seconds", runtimeWithLimit.Seconds())
runtimeWithoutLimit := b.Time("with basic bridged plugin", func() {
makeTCPClientInNS(hostNS.ShortName(), basicBridgeIP, basicBridgePort, packetInBytes)
})
Expect(runtimeWithLimit).To(BeNumerically(">", runtimeWithoutLimit+1000*time.Millisecond))
})
}, 1)
})
})

3
integration/testdata/basic-ptp.json vendored
View File

@ -6,7 +6,6 @@
"mtu": 512,
"ipam": {
"type": "host-local",
"subnet": "10.1.2.0/24",
"dataDir": "/tmp/basic-ptp-test"
"subnet": "10.1.2.0/24"
}
}

3
integration/testdata/chained-ptp-bandwidth.conflist vendored
View File

@ -8,8 +8,7 @@
"mtu": 512,
"ipam": {
"type": "host-local",
"subnet": "10.9.2.0/24",
"dataDir": "/tmp/chained-ptp-bandwidth-test"
"subnet": "10.9.2.0/24"
}
},
{

563
plugins/meta/bandwidth/bandwidth_config_test.go
View File

@ -1,563 +0,0 @@
// Copyright 2023 CNI authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import (
"fmt"
"math"
"net"
"syscall"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/vishvananda/netlink"
"github.com/containernetworking/cni/pkg/skel"
types100 "github.com/containernetworking/cni/pkg/types/100"
"github.com/containernetworking/plugins/pkg/ns"
"github.com/containernetworking/plugins/pkg/testutils"
)
var _ = Describe("bandwidth config test", func() {
var (
hostNs ns.NetNS
containerNs ns.NetNS
ifbDeviceName string
hostIfname string
containerIfname string
hostIP net.IP
containerIP net.IP
hostIfaceMTU int
)
BeforeEach(func() {
var err error
hostIfname = "host-veth"
containerIfname = "container-veth"
hostNs, err = testutils.NewNS()
Expect(err).NotTo(HaveOccurred())
containerNs, err = testutils.NewNS()
Expect(err).NotTo(HaveOccurred())
hostIP = net.IP{169, 254, 0, 1}
containerIP = net.IP{10, 254, 0, 1}
hostIfaceMTU = 1024
ifbDeviceName = "bwpa8eda89404b7"
createVeth(hostNs, hostIfname, containerNs, containerIfname, hostIP, containerIP, hostIfaceMTU)
})
AfterEach(func() {
Expect(containerNs.Close()).To(Succeed())
Expect(testutils.UnmountNS(containerNs)).To(Succeed())
Expect(hostNs.Close()).To(Succeed())
Expect(testutils.UnmountNS(hostNs)).To(Succeed())
})
// Bandwidth requires host-side interface info, and thus only
// supports 0.3.0 and later CNI versions
for _, ver := range []string{"0.3.0", "0.3.1", "0.4.0", "1.0.0"} {
// Redefine ver inside for scope so real value is picked up by each dynamically defined It()
// See Gingkgo's "Patterns for dynamically generating tests" documentation.
ver := ver
Describe("cmdADD", func() {
It(fmt.Sprintf("[%s] fails with invalid UnshapedSubnets", ver), func() {
conf := fmt.Sprintf(`{
"cniVersion": "%s",
"name": "cni-plugin-bandwidth-test",
"type": "bandwidth",
"ingressRate": 123,
"ingressBurst": 123,
"egressRate": 123,
"egressBurst": 123,
"unshapedSubnets": ["10.0.0.0/8", "hello"],
"prevResult": {
"interfaces": [
{
"name": "%s",
"sandbox": ""
},
{
"name": "%s",
"sandbox": "%s"
}
],
"ips": [
{
"version": "4",
"address": "%s/24",
"gateway": "10.0.0.1",
"interface": 1
}
],
"routes": []
}
}`, ver, hostIfname, containerIfname, containerNs.Path(), containerIP.String())
args := &skel.CmdArgs{
ContainerID: "dummy",
Netns: containerNs.Path(),
IfName: "eth0",
StdinData: []byte(conf),
}
Expect(hostNs.Do(func(netNS ns.NetNS) error {
defer GinkgoRecover()
_, _, err := testutils.CmdAdd(containerNs.Path(), args.ContainerID, "", []byte(conf), func() error { return cmdAdd(args) })
Expect(err).To(MatchError("bad subnet \"hello\" provided, details invalid CIDR address: hello"))
return nil
})).To(Succeed())
})
It(fmt.Sprintf("[%s] fails with invalid ShapedSubnets", ver), func() {
conf := fmt.Sprintf(`{
"cniVersion": "%s",
"name": "cni-plugin-bandwidth-test",
"type": "bandwidth",
"ingressRate": 123,
"ingressBurst": 123,
"egressRate": 123,
"egressBurst": 123,
"ShapedSubnets": ["10.0.0.0/8", "hello"],
"prevResult": {
"interfaces": [
{
"name": "%s",
"sandbox": ""
},
{
"name": "%s",
"sandbox": "%s"
}
],
"ips": [
{
"version": "4",
"address": "%s/24",
"gateway": "10.0.0.1",
"interface": 1
}
],
"routes": []
}
}`, ver, hostIfname, containerIfname, containerNs.Path(), containerIP.String())
args := &skel.CmdArgs{
ContainerID: "dummy",
Netns: containerNs.Path(),
IfName: "eth0",
StdinData: []byte(conf),
}
Expect(hostNs.Do(func(netNS ns.NetNS) error {
defer GinkgoRecover()
_, _, err := testutils.CmdAdd(containerNs.Path(), args.ContainerID, "", []byte(conf), func() error { return cmdAdd(args) })
Expect(err).To(MatchError("bad subnet \"hello\" provided, details invalid CIDR address: hello"))
return nil
})).To(Succeed())
})
It(fmt.Sprintf("[%s] fails with both ShapedSubnets and UnshapedSubnets specified", ver), func() {
conf := fmt.Sprintf(`{
"cniVersion": "%s",
"name": "cni-plugin-bandwidth-test",
"type": "bandwidth",
"ingressRate": 123,
"ingressBurst": 123,
"egressRate": 123,
"egressBurst": 123,
"shapedSubnets": ["10.0.0.0/8"],
"unshapedSubnets": ["192.168.0.0/16"],
"prevResult": {
"interfaces": [
{
"name": "%s",
"sandbox": ""
},
{
"name": "%s",
"sandbox": "%s"
}
],
"ips": [
{
"version": "4",
"address": "%s/24",
"gateway": "10.0.0.1",
"interface": 1
}
],
"routes": []
}
}`, ver, hostIfname, containerIfname, containerNs.Path(), containerIP.String())
args := &skel.CmdArgs{
ContainerID: "dummy",
Netns: containerNs.Path(),
IfName: "eth0",
StdinData: []byte(conf),
}
Expect(hostNs.Do(func(netNS ns.NetNS) error {
defer GinkgoRecover()
_, _, err := testutils.CmdAdd(containerNs.Path(), args.ContainerID, "", []byte(conf), func() error { return cmdAdd(args) })
Expect(err).To(MatchError("unshapedSubnets and shapedSubnets cannot be both specified, one of them should be discarded"))
return nil
})).To(Succeed())
})
It(fmt.Sprintf("[%s] fails an invalid ingress config", ver), func() {
conf := fmt.Sprintf(`{
"cniVersion": "%s",
"name": "cni-plugin-bandwidth-test",
"type": "bandwidth",
"ingressRate": 0,
"ingressBurst": 123,
"egressRate": 123,
"egressBurst": 123,
"prevResult": {
"interfaces": [
{
"name": "%s",
"sandbox": ""
},
{
"name": "%s",
"sandbox": "%s"
}
],
"ips": [
{
"version": "4",
"address": "%s/24",
"gateway": "10.0.0.1",
"interface": 1
}
],
"routes": []
}
}`, ver, hostIfname, containerIfname, containerNs.Path(), containerIP.String())
args := &skel.CmdArgs{
ContainerID: "dummy",
Netns: containerNs.Path(),
IfName: "eth0",
StdinData: []byte(conf),
}
Expect(hostNs.Do(func(netNS ns.NetNS) error {
defer GinkgoRecover()
_, _, err := testutils.CmdAdd(containerNs.Path(), args.ContainerID, "", []byte(conf), func() error { return cmdAdd(args) })
Expect(err).To(MatchError("if burst is set, rate must also be set"))
return nil
})).To(Succeed())
})
It(fmt.Sprintf("[%s] fails an invalid egress config", ver), func() {
conf := fmt.Sprintf(`{
"cniVersion": "%s",
"name": "cni-plugin-bandwidth-test",
"type": "bandwidth",
"ingressRate": 123,
"ingressBurst": 123,
"egressRate": 0,
"egressBurst": 123,
"prevResult": {
"interfaces": [
{
"name": "%s",
"sandbox": ""
},
{
"name": "%s",
"sandbox": "%s"
}
],
"ips": [
{
"version": "4",
"address": "%s/24",
"gateway": "10.0.0.1",
"interface": 1
}
],
"routes": []
}
}`, ver, hostIfname, containerIfname, containerNs.Path(), containerIP.String())
args := &skel.CmdArgs{
ContainerID: "dummy",
Netns: containerNs.Path(),
IfName: "eth0",
StdinData: []byte(conf),
}
Expect(hostNs.Do(func(netNS ns.NetNS) error {
defer GinkgoRecover()
_, _, err := testutils.CmdAdd(containerNs.Path(), args.ContainerID, "", []byte(conf), func() error { return cmdAdd(args) })
Expect(err).To(MatchError("if burst is set, rate must also be set"))
return nil
})).To(Succeed())
})
// Runtime config parameters are expected to be preempted by the global config ones whenever specified
It(fmt.Sprintf("[%s] should apply static config when both static config and runtime config exist", ver), func() {
conf := fmt.Sprintf(`{
"cniVersion": "%s",
"name": "cni-plugin-bandwidth-test",
"type": "bandwidth",
"ingressRate": 0,
"ingressBurst": 0,
"egressRate": 123,
"egressBurst": 123,
"unshapedSubnets": ["192.168.0.0/24"],
"runtimeConfig": {
"bandWidth": {
"ingressRate": 8,
"ingressBurst": 8,
"egressRate": 16,
"egressBurst": 9,
"unshapedSubnets": ["10.0.0.0/8", "fd00:db8:abcd:1234:e000::/68"]
}
},
"prevResult": {
"interfaces": [
{
"name": "%s",
"sandbox": ""
},
{
"name": "%s",
"sandbox": "%s"
}
],
"ips": [
{
"version": "4",
"address": "%s/24",
"gateway": "10.0.0.1",
"interface": 1
}
],
"routes": []
}
}`, ver, hostIfname, containerIfname, containerNs.Path(), containerIP.String())
args := &skel.CmdArgs{
ContainerID: "dummy",
Netns: containerNs.Path(),
IfName: containerIfname,
StdinData: []byte(conf),
}
Expect(hostNs.Do(func(netNS ns.NetNS) error {
defer GinkgoRecover()
r, out, err := testutils.CmdAdd(containerNs.Path(), args.ContainerID, "", []byte(conf), func() error { return cmdAdd(args) })
Expect(err).NotTo(HaveOccurred(), string(out))
result, err := types100.GetResult(r)
Expect(err).NotTo(HaveOccurred())
Expect(result.Interfaces).To(HaveLen(3))
Expect(result.Interfaces[2].Name).To(Equal(ifbDeviceName))
Expect(result.Interfaces[2].Sandbox).To(Equal(""))
ifbLink, err := netlink.LinkByName(ifbDeviceName)
Expect(err).NotTo(HaveOccurred())
Expect(ifbLink.Attrs().MTU).To(Equal(hostIfaceMTU))
qdiscs, err := netlink.QdiscList(ifbLink)
Expect(err).NotTo(HaveOccurred())
Expect(qdiscs).To(HaveLen(1))
Expect(qdiscs[0].Attrs().LinkIndex).To(Equal(ifbLink.Attrs().Index))
Expect(qdiscs[0]).To(BeAssignableToTypeOf(&netlink.Htb{}))
Expect(qdiscs[0].(*netlink.Htb).Defcls).To(Equal(uint32(ShapedClassMinorID)))
classes, err := netlink.ClassList(ifbLink, qdiscs[0].Attrs().Handle)
Expect(err).NotTo(HaveOccurred())
Expect(classes).To(HaveLen(2))
// Uncapped class
Expect(classes[0]).To(BeAssignableToTypeOf(&netlink.HtbClass{}))
Expect(classes[0].(*netlink.HtbClass).Handle).To(Equal(netlink.MakeHandle(1, 1)))
Expect(classes[0].(*netlink.HtbClass).Rate).To(Equal(UncappedRate))
Expect(classes[0].(*netlink.HtbClass).Buffer).To(Equal(uint32(0)))
Expect(classes[0].(*netlink.HtbClass).Ceil).To(Equal(UncappedRate))
Expect(classes[0].(*netlink.HtbClass).Cbuffer).To(Equal(uint32(0)))
// Class with traffic shapping settings
Expect(classes[1]).To(BeAssignableToTypeOf(&netlink.HtbClass{}))
Expect(classes[1].(*netlink.HtbClass).Handle).To(Equal(netlink.MakeHandle(1, uint16(qdiscs[0].(*netlink.Htb).Defcls))))
Expect(classes[1].(*netlink.HtbClass).Rate).To(Equal(uint64(15)))
// Expect(classes[1].(*netlink.HtbClass).Buffer).To(Equal(uint32(7812500)))
Expect(classes[1].(*netlink.HtbClass).Ceil).To(Equal(uint64(30)))
// Expect(classes[1].(*netlink.HtbClass).Cbuffer).To(Equal(uint32(0)))
filters, err := netlink.FilterList(ifbLink, qdiscs[0].Attrs().Handle)
Expect(err).NotTo(HaveOccurred())
Expect(filters).To(HaveLen(1))
// traffic to 192.168.0.0/24 redirected to uncapped class
Expect(filters[0]).To(BeAssignableToTypeOf(&netlink.U32{}))
Expect(filters[0].(*netlink.U32).Actions).To(BeEmpty())
Expect(filters[0].Attrs().Protocol).To(Equal(uint16(syscall.ETH_P_IP)))
Expect(filters[0].Attrs().LinkIndex).To(Equal(ifbLink.Attrs().Index))
Expect(filters[0].Attrs().Priority).To(Equal(uint16(16)))
Expect(filters[0].Attrs().Parent).To(Equal(qdiscs[0].Attrs().Handle))
Expect(filters[0].(*netlink.U32).ClassId).To(Equal(netlink.MakeHandle(1, 1)))
filterSel := filters[0].(*netlink.U32).Sel
Expect(filterSel).To(BeAssignableToTypeOf(&netlink.TcU32Sel{}))
Expect(filterSel.Flags).To(Equal(uint8(netlink.TC_U32_TERMINAL)))
Expect(filterSel.Keys).To(HaveLen(1))
Expect(filterSel.Nkeys).To(Equal(uint8(1)))
// The filter should match to 192.168.0.0/24 dst address in other words it should be:
// match c0a80000/ffffff00 at 16
selKey := filterSel.Keys[0]
Expect(selKey.Val).To(Equal(uint32(192*math.Pow(256, 3) + 168*math.Pow(256, 2))))
Expect(selKey.Off).To(Equal(int32(16)))
Expect(selKey.Mask).To(Equal(uint32(255*math.Pow(256, 3) + 255*math.Pow(256, 2) + 255*256)))
hostVethLink, err := netlink.LinkByName(hostIfname)
Expect(err).NotTo(HaveOccurred())
qdiscFilters, err := netlink.FilterList(hostVethLink, netlink.MakeHandle(0xffff, 0))
Expect(err).NotTo(HaveOccurred())
Expect(qdiscFilters).To(HaveLen(1))
Expect(qdiscFilters[0].(*netlink.U32).Actions[0].(*netlink.MirredAction).Ifindex).To(Equal(ifbLink.Attrs().Index))
return nil
})).To(Succeed())
// Container ingress (host egress)
Expect(hostNs.Do(func(n ns.NetNS) error {
defer GinkgoRecover()
vethLink, err := netlink.LinkByName(hostIfname)
Expect(err).NotTo(HaveOccurred())
qdiscs, err := netlink.QdiscList(vethLink)
Expect(err).NotTo(HaveOccurred())
// No ingress QoS just mirroring
Expect(qdiscs).To(HaveLen(2))
Expect(qdiscs[0].Attrs().LinkIndex).To(Equal(vethLink.Attrs().Index))
Expect(qdiscs[0]).NotTo(BeAssignableToTypeOf(&netlink.Htb{}))
Expect(qdiscs[1]).NotTo(BeAssignableToTypeOf(&netlink.Htb{}))
return nil
})).To(Succeed())
})
It(fmt.Sprintf("[%s] should apply static config when both static config and runtime config exist (bad config example)", ver), func() {
conf := fmt.Sprintf(`{
"cniVersion": "%s",
"name": "cni-plugin-bandwidth-test",
"type": "bandwidth",
"ingressRate": 0,
"ingressBurst": 123,
"egressRate": 123,
"egressBurst": 123,
"runtimeConfig": {
"bandWidth": {
"ingressRate": 8,
"ingressBurst": 8,
"egressRate": 16,
"egressBurst": 9
}
},
"prevResult": {
"interfaces": [
{
"name": "%s",
"sandbox": ""
},
{
"name": "%s",
"sandbox": "%s"
}
],
"ips": [
{
"version": "4",
"address": "%s/24",
"gateway": "10.0.0.1",
"interface": 1
}
],
"routes": []
}
}`, ver, hostIfname, containerIfname, containerNs.Path(), containerIP.String())
args := &skel.CmdArgs{
ContainerID: "dummy",
Netns: containerNs.Path(),
IfName: "eth0",
StdinData: []byte(conf),
}
Expect(hostNs.Do(func(netNS ns.NetNS) error {
defer GinkgoRecover()
_, _, err := testutils.CmdAdd(containerNs.Path(), args.ContainerID, "", []byte(conf), func() error { return cmdAdd(args) })
Expect(err).To(MatchError("if burst is set, rate must also be set"))
return nil
})).To(Succeed())
})
})
}
Describe("Validating input", func() {
It("Should allow only 4GB burst rate", func() {
err := validateRateAndBurst(5000, 4*1024*1024*1024*8-16) // 2 bytes less than the max should pass
Expect(err).NotTo(HaveOccurred())
err = validateRateAndBurst(5000, 4*1024*1024*1024*8) // we're 1 bit above MaxUint32
Expect(err).To(HaveOccurred())
err = validateRateAndBurst(0, 1)
Expect(err).To(HaveOccurred())
err = validateRateAndBurst(1, 0)
Expect(err).To(HaveOccurred())
err = validateRateAndBurst(0, 0)
Expect(err).NotTo(HaveOccurred())
})
It("Should fail if both ShapedSubnets and UnshapedSubnets are specified", func() {
err := validateSubnets([]string{"10.0.0.0/8"}, []string{"192.168.0.0/24"})
Expect(err).To(HaveOccurred())
})
It("Should fail if specified UnshapedSubnets are not valid CIDRs", func() {
err := validateSubnets([]string{"10.0.0.0/8", "hello"}, []string{})
Expect(err).To(HaveOccurred())
})
It("Should fail if specified ShapedSubnets are not valid CIDRs", func() {
err := validateSubnets([]string{}, []string{"10.0.0.0/8", "hello"})
Expect(err).To(HaveOccurred())
})
})
})

1659
plugins/meta/bandwidth/bandwidth_linux_test.go
View File

File diff suppressed because it is too large Load Diff

824
plugins/meta/bandwidth/bandwidth_measure_linux_test.go
View File

@ -1,824 +0,0 @@
// Copyright 2023 CNI authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import (
"context"
"encoding/json"
"fmt"
"log"
"net"
"os"
"time"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/onsi/gomega/gexec"
"github.com/containernetworking/cni/pkg/invoke"
"github.com/containernetworking/cni/pkg/skel"
"github.com/containernetworking/cni/pkg/types"
types100 "github.com/containernetworking/cni/pkg/types/100"
"github.com/containernetworking/plugins/pkg/ns"
"github.com/containernetworking/plugins/pkg/testutils"
)
var _ = Describe("bandwidth measure test", func() {
var (
hostNs ns.NetNS
containerNs ns.NetNS
hostIfname string
containerIfname string
hostIP net.IP
containerIP net.IP
hostIfaceMTU int
)
BeforeEach(func() {
var err error
hostIfname = "host-veth"
containerIfname = "container-veth"
hostNs, err = testutils.NewNS()
Expect(err).NotTo(HaveOccurred())
containerNs, err = testutils.NewNS()
Expect(err).NotTo(HaveOccurred())
hostIP = net.IP{169, 254, 0, 1}
containerIP = net.IP{10, 254, 0, 1}
hostIfaceMTU = 1024
createVeth(hostNs, hostIfname, containerNs, containerIfname, hostIP, containerIP, hostIfaceMTU)
})
AfterEach(func() {
Expect(containerNs.Close()).To(Succeed())
Expect(testutils.UnmountNS(containerNs)).To(Succeed())
Expect(hostNs.Close()).To(Succeed())
Expect(testutils.UnmountNS(hostNs)).To(Succeed())
})
// Bandwidth requires host-side interface info, and thus only
// supports 0.3.0 and later CNI versions
for _, ver := range []string{"0.3.0", "0.3.1", "0.4.0", "1.0.0"} {
Describe(fmt.Sprintf("[%s] QoS effective", ver), func() {
Context(fmt.Sprintf("[%s] when chaining bandwidth plugin with PTP", ver), func() {
var ptpConf string
var rateInBits uint64
var burstInBits uint64
var packetInBytes int
var containerWithoutQoSNS ns.NetNS
var containerWithQoSNS ns.NetNS
var portServerWithQoS int
var portServerWithoutQoS int
var containerWithQoSRes types.Result
var containerWithoutQoSRes types.Result
var echoServerWithQoS *gexec.Session
var echoServerWithoutQoS *gexec.Session
var dataDir string
BeforeEach(func() {
rateInBytes := 1000
rateInBits = uint64(rateInBytes * 8)
burstInBits = rateInBits * 2
// NOTE: Traffic shapping is not that precise at low rates, would be better to use higher rates + simple time+netcat for data transfer, rather than the provided
// client/server bin (limited to small amount of data)
packetInBytes = rateInBytes * 3
var err error
dataDir, err = os.MkdirTemp("", "bandwidth_linux_test")
Expect(err).NotTo(HaveOccurred())
ptpConf = fmt.Sprintf(`{
"cniVersion": "%s",
"name": "myBWnet",
"type": "ptp",
"ipMasq": true,
"mtu": 512,
"ipam": {
"type": "host-local",
"subnet": "10.1.2.0/24",
"dataDir": "%s"
}
}`, ver, dataDir)
const (
containerWithQoSIFName = "ptp0"
containerWithoutQoSIFName = "ptp1"
)
containerWithQoSNS, err = testutils.NewNS()
Expect(err).NotTo(HaveOccurred())
containerWithoutQoSNS, err = testutils.NewNS()
Expect(err).NotTo(HaveOccurred())
By("create two containers, and use the bandwidth plugin on one of them")
Expect(hostNs.Do(func(ns.NetNS) error {
defer GinkgoRecover()
containerWithQoSRes, _, err = testutils.CmdAdd(containerWithQoSNS.Path(), "dummy", containerWithQoSIFName, []byte(ptpConf), func() error {
r, err := invoke.DelegateAdd(context.TODO(), "ptp", []byte(ptpConf), nil)
Expect(err).NotTo(HaveOccurred())
Expect(r.Print()).To(Succeed())
return err
})
Expect(err).NotTo(HaveOccurred())
containerWithoutQoSRes, _, err = testutils.CmdAdd(containerWithoutQoSNS.Path(), "dummy2", containerWithoutQoSIFName, []byte(ptpConf), func() error {
r, err := invoke.DelegateAdd(context.TODO(), "ptp", []byte(ptpConf), nil)
Expect(err).NotTo(HaveOccurred())
Expect(r.Print()).To(Succeed())
return err
})
Expect(err).NotTo(HaveOccurred())
containerWithQoSResult, err := types100.GetResult(containerWithQoSRes)
Expect(err).NotTo(HaveOccurred())
bandwidthPluginConf := &PluginConf{}
err = json.Unmarshal([]byte(ptpConf), &bandwidthPluginConf)
Expect(err).NotTo(HaveOccurred())
bandwidthPluginConf.RuntimeConfig.Bandwidth = &BandwidthEntry{
IngressBurst: burstInBits,
IngressRate: rateInBits,
EgressBurst: burstInBits,
EgressRate: rateInBits,
}
bandwidthPluginConf.Type = "bandwidth"
newConfBytes, err := buildOneConfig(ver, bandwidthPluginConf, containerWithQoSResult)
Expect(err).NotTo(HaveOccurred())
args := &skel.CmdArgs{
ContainerID: "dummy3",
Netns: containerWithQoSNS.Path(),
IfName: containerWithQoSIFName,
StdinData: newConfBytes,
}
result, out, err := testutils.CmdAdd(containerWithQoSNS.Path(), args.ContainerID, "", newConfBytes, func() error { return cmdAdd(args) })
Expect(err).NotTo(HaveOccurred(), string(out))
if testutils.SpecVersionHasCHECK(ver) {
// Do CNI Check
checkConf := &PluginConf{}
err = json.Unmarshal([]byte(ptpConf), &checkConf)
Expect(err).NotTo(HaveOccurred())
checkConf.RuntimeConfig.Bandwidth = &BandwidthEntry{
IngressBurst: burstInBits,
IngressRate: rateInBits,
EgressBurst: burstInBits,
EgressRate: rateInBits,
}
checkConf.Type = "bandwidth"
newCheckBytes, err := buildOneConfig(ver, checkConf, result)
Expect(err).NotTo(HaveOccurred())
args = &skel.CmdArgs{
ContainerID: "dummy3",
Netns: containerWithQoSNS.Path(),
IfName: containerWithQoSIFName,
StdinData: newCheckBytes,
}
err = testutils.CmdCheck(containerWithQoSNS.Path(), args.ContainerID, "", func() error { return cmdCheck(args) })
Expect(err).NotTo(HaveOccurred())
}
return nil
})).To(Succeed())
By("starting a tcp server on both containers")
portServerWithQoS, echoServerWithQoS = startEchoServerInNamespace(containerWithQoSNS)
portServerWithoutQoS, echoServerWithoutQoS = startEchoServerInNamespace(containerWithoutQoSNS)
})
AfterEach(func() {
Expect(os.RemoveAll(dataDir)).To(Succeed())
Expect(containerWithQoSNS.Close()).To(Succeed())
Expect(testutils.UnmountNS(containerWithQoSNS)).To(Succeed())
Expect(containerWithoutQoSNS.Close()).To(Succeed())
Expect(testutils.UnmountNS(containerWithoutQoSNS)).To(Succeed())
if echoServerWithoutQoS != nil {
echoServerWithoutQoS.Kill()
}
if echoServerWithQoS != nil {
echoServerWithQoS.Kill()
}
})
It("limits ingress traffic on veth device", func() {
var runtimeWithLimit time.Duration
var runtimeWithoutLimit time.Duration
By("gather timing statistics about both containers")
By("sending tcp traffic to the container that has traffic shaped", func() {
start := time.Now()
result, err := types100.GetResult(containerWithQoSRes)
Expect(err).NotTo(HaveOccurred())
makeTCPClientInNS(hostNs.Path(), result.IPs[0].Address.IP.String(), portServerWithQoS, packetInBytes)
end := time.Now()
runtimeWithLimit = end.Sub(start)
log.Printf("Elapsed with qos %.2f", runtimeWithLimit.Seconds())
})
By("sending tcp traffic to the container that does not have traffic shaped", func() {
start := time.Now()
result, err := types100.GetResult(containerWithoutQoSRes)
Expect(err).NotTo(HaveOccurred())
makeTCPClientInNS(hostNs.Path(), result.IPs[0].Address.IP.String(), portServerWithoutQoS, packetInBytes)
end := time.Now()
runtimeWithoutLimit = end.Sub(start)
log.Printf("Elapsed without qos %.2f", runtimeWithoutLimit.Seconds())
})
Expect(runtimeWithLimit).To(BeNumerically(">", runtimeWithoutLimit+1000*time.Millisecond))
})
})
})
Context(fmt.Sprintf("[%s] when chaining bandwidth plugin with PTP and excluding specific subnets from traffic", ver), func() {
var ptpConf string
var rateInBits uint64
var burstInBits uint64
var packetInBytes int
var containerWithoutQoSNS ns.NetNS
var containerWithQoSNS ns.NetNS
var portServerWithQoS int
var portServerWithoutQoS int
var containerWithQoSRes types.Result
var containerWithoutQoSRes types.Result
var echoServerWithQoS *gexec.Session
var echoServerWithoutQoS *gexec.Session
var dataDir string
BeforeEach(func() {
rateInBytes := 1000
rateInBits = uint64(rateInBytes * 8)
burstInBits = rateInBits * 2
unshapedSubnets := []string{"10.1.2.0/24"}
// NOTE: Traffic shapping is not that precise at low rates, would be better to use higher rates + simple time+netcat for data transfer, rather than the provided
// client/server bin (limited to small amount of data)
packetInBytes = rateInBytes * 3
var err error
dataDir, err = os.MkdirTemp("", "bandwidth_linux_test")
Expect(err).NotTo(HaveOccurred())
ptpConf = fmt.Sprintf(`{
"cniVersion": "%s",
"name": "myBWnet",
"type": "ptp",
"ipMasq": true,
"mtu": 512,
"ipam": {
"type": "host-local",
"subnet": "10.1.2.0/24",
"dataDir": "%s"
}
}`, ver, dataDir)
const (
containerWithQoSIFName = "ptp0"
containerWithoutQoSIFName = "ptp1"
)
containerWithQoSNS, err = testutils.NewNS()
Expect(err).NotTo(HaveOccurred())
containerWithoutQoSNS, err = testutils.NewNS()
Expect(err).NotTo(HaveOccurred())
By("create two containers, and use the bandwidth plugin on one of them")
Expect(hostNs.Do(func(ns.NetNS) error {
defer GinkgoRecover()
containerWithQoSRes, _, err = testutils.CmdAdd(containerWithQoSNS.Path(), "dummy", containerWithQoSIFName, []byte(ptpConf), func() error {
r, err := invoke.DelegateAdd(context.TODO(), "ptp", []byte(ptpConf), nil)
Expect(err).NotTo(HaveOccurred())
Expect(r.Print()).To(Succeed())
return err
})
Expect(err).NotTo(HaveOccurred())
containerWithoutQoSRes, _, err = testutils.CmdAdd(containerWithoutQoSNS.Path(), "dummy2", containerWithoutQoSIFName, []byte(ptpConf), func() error {
r, err := invoke.DelegateAdd(context.TODO(), "ptp", []byte(ptpConf), nil)
Expect(err).NotTo(HaveOccurred())
Expect(r.Print()).To(Succeed())
return err
})
Expect(err).NotTo(HaveOccurred())
containerWithQoSResult, err := types100.GetResult(containerWithQoSRes)
Expect(err).NotTo(HaveOccurred())
bandwidthPluginConf := &PluginConf{}
err = json.Unmarshal([]byte(ptpConf), &bandwidthPluginConf)
Expect(err).NotTo(HaveOccurred())
bandwidthPluginConf.RuntimeConfig.Bandwidth = &BandwidthEntry{
IngressBurst: burstInBits,
IngressRate: rateInBits,
EgressBurst: burstInBits,
EgressRate: rateInBits,
UnshapedSubnets: unshapedSubnets,
}
bandwidthPluginConf.Type = "bandwidth"
newConfBytes, err := buildOneConfig(ver, bandwidthPluginConf, containerWithQoSResult)
Expect(err).NotTo(HaveOccurred())
args := &skel.CmdArgs{
ContainerID: "dummy3",
Netns: containerWithQoSNS.Path(),
IfName: containerWithQoSIFName,
StdinData: newConfBytes,
}
result, out, err := testutils.CmdAdd(containerWithQoSNS.Path(), args.ContainerID, "", newConfBytes, func() error { return cmdAdd(args) })
Expect(err).NotTo(HaveOccurred(), string(out))
if testutils.SpecVersionHasCHECK(ver) {
// Do CNI Check
checkConf := &PluginConf{}
err = json.Unmarshal([]byte(ptpConf), &checkConf)
Expect(err).NotTo(HaveOccurred())
checkConf.RuntimeConfig.Bandwidth = &BandwidthEntry{
IngressBurst: burstInBits,
IngressRate: rateInBits,
EgressBurst: burstInBits,
EgressRate: rateInBits,
UnshapedSubnets: unshapedSubnets,
}
checkConf.Type = "bandwidth"
newCheckBytes, err := buildOneConfig(ver, checkConf, result)
Expect(err).NotTo(HaveOccurred())
args = &skel.CmdArgs{
ContainerID: "dummy3",
Netns: containerWithQoSNS.Path(),
IfName: containerWithQoSIFName,
StdinData: newCheckBytes,
}
err = testutils.CmdCheck(containerWithQoSNS.Path(), args.ContainerID, "", func() error { return cmdCheck(args) })
Expect(err).NotTo(HaveOccurred())
}
return nil
})).To(Succeed())
By("starting a tcp server on both containers")
portServerWithQoS, echoServerWithQoS = startEchoServerInNamespace(containerWithQoSNS)
portServerWithoutQoS, echoServerWithoutQoS = startEchoServerInNamespace(containerWithoutQoSNS)
})
AfterEach(func() {
Expect(os.RemoveAll(dataDir)).To(Succeed())
Expect(containerWithQoSNS.Close()).To(Succeed())
Expect(testutils.UnmountNS(containerWithQoSNS)).To(Succeed())
Expect(containerWithoutQoSNS.Close()).To(Succeed())
Expect(testutils.UnmountNS(containerWithoutQoSNS)).To(Succeed())
if echoServerWithoutQoS != nil {
echoServerWithoutQoS.Kill()
}
if echoServerWithQoS != nil {
echoServerWithQoS.Kill()
}
})
It("does not limits ingress traffic on veth device coming from 10.1.2.0/24", func() {
var runtimeWithLimit time.Duration
var runtimeWithoutLimit time.Duration
By("gather timing statistics about both containers")
By("sending tcp traffic to the container that has traffic shaped", func() {
start := time.Now()
result, err := types100.GetResult(containerWithQoSRes)
Expect(err).NotTo(HaveOccurred())
makeTCPClientInNS(hostNs.Path(), result.IPs[0].Address.IP.String(), portServerWithQoS, packetInBytes)
end := time.Now()
runtimeWithLimit = end.Sub(start)
log.Printf("Elapsed with qos %.2f", runtimeWithLimit.Seconds())
})
By("sending tcp traffic to the container that does not have traffic shaped", func() {
start := time.Now()
result, err := types100.GetResult(containerWithoutQoSRes)
Expect(err).NotTo(HaveOccurred())
makeTCPClientInNS(hostNs.Path(), result.IPs[0].Address.IP.String(), portServerWithoutQoS, packetInBytes)
end := time.Now()
runtimeWithoutLimit = end.Sub(start)
log.Printf("Elapsed without qos %.2f", runtimeWithoutLimit.Seconds())
})
Expect(runtimeWithLimit - runtimeWithoutLimit).To(BeNumerically("<", 100*time.Millisecond))
})
})
Context(fmt.Sprintf("[%s] when chaining bandwidth plugin with PTP and only including specific subnets in traffic shapping (not including the main ns one)", ver), func() {
var ptpConf string
var rateInBits uint64
var burstInBits uint64
var packetInBytes int
var containerWithoutQoSNS ns.NetNS
var containerWithQoSNS ns.NetNS
var portServerWithQoS int
var portServerWithoutQoS int
var containerWithQoSRes types.Result
var containerWithoutQoSRes types.Result
var echoServerWithQoS *gexec.Session
var echoServerWithoutQoS *gexec.Session
var dataDir string
BeforeEach(func() {
rateInBytes := 1000
rateInBits = uint64(rateInBytes * 8)
burstInBits = rateInBits * 2
shapedSubnets := []string{"10.2.2.0/24"}
// NOTE: Traffic shapping is not that precise at low rates, would be better to use higher rates + simple time+netcat for data transfer, rather than the provided
// client/server bin (limited to small amount of data)
packetInBytes = rateInBytes * 3
var err error
dataDir, err = os.MkdirTemp("", "bandwidth_linux_test")
Expect(err).NotTo(HaveOccurred())
ptpConf = fmt.Sprintf(`{
"cniVersion": "%s",
"name": "myBWnet",
"type": "ptp",
"ipMasq": true,
"mtu": 512,
"ipam": {
"type": "host-local",
"subnet": "10.1.2.0/24",
"dataDir": "%s"
}
}`, ver, dataDir)
const (
containerWithQoSIFName = "ptp0"
containerWithoutQoSIFName = "ptp1"
)
containerWithQoSNS, err = testutils.NewNS()
Expect(err).NotTo(HaveOccurred())
containerWithoutQoSNS, err = testutils.NewNS()
Expect(err).NotTo(HaveOccurred())
By("create two containers, and use the bandwidth plugin on one of them")
Expect(hostNs.Do(func(ns.NetNS) error {
defer GinkgoRecover()
containerWithQoSRes, _, err = testutils.CmdAdd(containerWithQoSNS.Path(), "dummy", containerWithQoSIFName, []byte(ptpConf), func() error {
r, err := invoke.DelegateAdd(context.TODO(), "ptp", []byte(ptpConf), nil)
Expect(err).NotTo(HaveOccurred())
Expect(r.Print()).To(Succeed())
return err
})
Expect(err).NotTo(HaveOccurred())
containerWithoutQoSRes, _, err = testutils.CmdAdd(containerWithoutQoSNS.Path(), "dummy2", containerWithoutQoSIFName, []byte(ptpConf), func() error {
r, err := invoke.DelegateAdd(context.TODO(), "ptp", []byte(ptpConf), nil)
Expect(err).NotTo(HaveOccurred())
Expect(r.Print()).To(Succeed())
return err
})
Expect(err).NotTo(HaveOccurred())
containerWithQoSResult, err := types100.GetResult(containerWithQoSRes)
Expect(err).NotTo(HaveOccurred())
bandwidthPluginConf := &PluginConf{}
err = json.Unmarshal([]byte(ptpConf), &bandwidthPluginConf)
Expect(err).NotTo(HaveOccurred())
bandwidthPluginConf.RuntimeConfig.Bandwidth = &BandwidthEntry{
IngressBurst: burstInBits,
IngressRate: rateInBits,
EgressBurst: burstInBits,
EgressRate: rateInBits,
ShapedSubnets: shapedSubnets,
}
bandwidthPluginConf.Type = "bandwidth"
newConfBytes, err := buildOneConfig(ver, bandwidthPluginConf, containerWithQoSResult)
Expect(err).NotTo(HaveOccurred())
args := &skel.CmdArgs{
ContainerID: "dummy3",
Netns: containerWithQoSNS.Path(),
IfName: containerWithQoSIFName,
StdinData: newConfBytes,
}
result, out, err := testutils.CmdAdd(containerWithQoSNS.Path(), args.ContainerID, "", newConfBytes, func() error { return cmdAdd(args) })
Expect(err).NotTo(HaveOccurred(), string(out))
if testutils.SpecVersionHasCHECK(ver) {
// Do CNI Check
checkConf := &PluginConf{}
err = json.Unmarshal([]byte(ptpConf), &checkConf)
Expect(err).NotTo(HaveOccurred())
checkConf.RuntimeConfig.Bandwidth = &BandwidthEntry{
IngressBurst: burstInBits,
IngressRate: rateInBits,
EgressBurst: burstInBits,
EgressRate: rateInBits,
ShapedSubnets: shapedSubnets,
}
checkConf.Type = "bandwidth"
newCheckBytes, err := buildOneConfig(ver, checkConf, result)
Expect(err).NotTo(HaveOccurred())
args = &skel.CmdArgs{
ContainerID: "dummy3",
Netns: containerWithQoSNS.Path(),
IfName: containerWithQoSIFName,
StdinData: newCheckBytes,
}
err = testutils.CmdCheck(containerWithQoSNS.Path(), args.ContainerID, "", func() error { return cmdCheck(args) })
Expect(err).NotTo(HaveOccurred())
}
return nil
})).To(Succeed())
By("starting a tcp server on both containers")
portServerWithQoS, echoServerWithQoS = startEchoServerInNamespace(containerWithQoSNS)
portServerWithoutQoS, echoServerWithoutQoS = startEchoServerInNamespace(containerWithoutQoSNS)
})
AfterEach(func() {
Expect(os.RemoveAll(dataDir)).To(Succeed())
Expect(containerWithQoSNS.Close()).To(Succeed())
Expect(testutils.UnmountNS(containerWithQoSNS)).To(Succeed())
Expect(containerWithoutQoSNS.Close()).To(Succeed())
Expect(testutils.UnmountNS(containerWithoutQoSNS)).To(Succeed())
if echoServerWithoutQoS != nil {
echoServerWithoutQoS.Kill()
}
if echoServerWithQoS != nil {
echoServerWithQoS.Kill()
}
})
It("does not limit ingress traffic on veth device coming from non included subnets", func() {
var runtimeWithLimit time.Duration
var runtimeWithoutLimit time.Duration
By("gather timing statistics about both containers")
By("sending tcp traffic to the container that has traffic shaped", func() {
start := time.Now()
result, err := types100.GetResult(containerWithQoSRes)
Expect(err).NotTo(HaveOccurred())
makeTCPClientInNS(hostNs.Path(), result.IPs[0].Address.IP.String(), portServerWithQoS, packetInBytes)
end := time.Now()
runtimeWithLimit = end.Sub(start)
log.Printf("Elapsed with qos %.2f", runtimeWithLimit.Seconds())
})
By("sending tcp traffic to the container that does not have traffic shaped", func() {
start := time.Now()
result, err := types100.GetResult(containerWithoutQoSRes)
Expect(err).NotTo(HaveOccurred())
makeTCPClientInNS(hostNs.Path(), result.IPs[0].Address.IP.String(), portServerWithoutQoS, packetInBytes)
end := time.Now()
runtimeWithoutLimit = end.Sub(start)
log.Printf("Elapsed without qos %.2f", runtimeWithoutLimit.Seconds())
})
Expect(runtimeWithLimit - runtimeWithoutLimit).To(BeNumerically("<", 100*time.Millisecond))
})
})
Context(fmt.Sprintf("[%s] when chaining bandwidth plugin with PTP and only including specific subnets in traffic shapping (including the main ns one)", ver), func() {
var ptpConf string
var rateInBits uint64
var burstInBits uint64
var packetInBytes int
var containerWithoutQoSNS ns.NetNS
var containerWithQoSNS ns.NetNS
var portServerWithQoS int
var portServerWithoutQoS int
var containerWithQoSRes types.Result
var containerWithoutQoSRes types.Result
var echoServerWithQoS *gexec.Session
var echoServerWithoutQoS *gexec.Session
var dataDir string
BeforeEach(func() {
rateInBytes := 1000
rateInBits = uint64(rateInBytes * 8)
burstInBits = rateInBits * 2
shapedSubnets := []string{"10.1.2.1/32"}
// NOTE: Traffic shapping is not that precise at low rates, would be better to use higher rates + simple time+netcat for data transfer, rather than the provided
// client/server bin (limited to small amount of data)
packetInBytes = rateInBytes * 3
var err error
dataDir, err = os.MkdirTemp("", "bandwidth_linux_test")
Expect(err).NotTo(HaveOccurred())
ptpConf = fmt.Sprintf(`{
"cniVersion": "%s",
"name": "myBWnet",
"type": "ptp",
"ipMasq": true,
"mtu": 512,
"ipam": {
"type": "host-local",
"subnet": "10.1.2.0/24",
"dataDir": "%s"
}
}`, ver, dataDir)
const (
containerWithQoSIFName = "ptp0"
containerWithoutQoSIFName = "ptp1"
)
containerWithQoSNS, err = testutils.NewNS()
Expect(err).NotTo(HaveOccurred())
containerWithoutQoSNS, err = testutils.NewNS()
Expect(err).NotTo(HaveOccurred())
By("create two containers, and use the bandwidth plugin on one of them")
Expect(hostNs.Do(func(ns.NetNS) error {
defer GinkgoRecover()
containerWithQoSRes, _, err = testutils.CmdAdd(containerWithQoSNS.Path(), "dummy", containerWithQoSIFName, []byte(ptpConf), func() error {
r, err := invoke.DelegateAdd(context.TODO(), "ptp", []byte(ptpConf), nil)
Expect(err).NotTo(HaveOccurred())
Expect(r.Print()).To(Succeed())
return err
})
Expect(err).NotTo(HaveOccurred())
containerWithoutQoSRes, _, err = testutils.CmdAdd(containerWithoutQoSNS.Path(), "dummy2", containerWithoutQoSIFName, []byte(ptpConf), func() error {
r, err := invoke.DelegateAdd(context.TODO(), "ptp", []byte(ptpConf), nil)
Expect(err).NotTo(HaveOccurred())
Expect(r.Print()).To(Succeed())
return err
})
Expect(err).NotTo(HaveOccurred())
containerWithQoSResult, err := types100.GetResult(containerWithQoSRes)
Expect(err).NotTo(HaveOccurred())
bandwidthPluginConf := &PluginConf{}
err = json.Unmarshal([]byte(ptpConf), &bandwidthPluginConf)
Expect(err).NotTo(HaveOccurred())
bandwidthPluginConf.RuntimeConfig.Bandwidth = &BandwidthEntry{
IngressBurst: burstInBits,
IngressRate: rateInBits,
EgressBurst: burstInBits,
EgressRate: rateInBits,
ShapedSubnets: shapedSubnets,
}
bandwidthPluginConf.Type = "bandwidth"
newConfBytes, err := buildOneConfig(ver, bandwidthPluginConf, containerWithQoSResult)
Expect(err).NotTo(HaveOccurred())
args := &skel.CmdArgs{
ContainerID: "dummy3",
Netns: containerWithQoSNS.Path(),
IfName: containerWithQoSIFName,
StdinData: newConfBytes,
}
result, out, err := testutils.CmdAdd(containerWithQoSNS.Path(), args.ContainerID, "", newConfBytes, func() error { return cmdAdd(args) })
Expect(err).NotTo(HaveOccurred(), string(out))
if testutils.SpecVersionHasCHECK(ver) {
// Do CNI Check
checkConf := &PluginConf{}
err = json.Unmarshal([]byte(ptpConf), &checkConf)
Expect(err).NotTo(HaveOccurred())
checkConf.RuntimeConfig.Bandwidth = &BandwidthEntry{
IngressBurst: burstInBits,
IngressRate: rateInBits,
EgressBurst: burstInBits,
EgressRate: rateInBits,
ShapedSubnets: shapedSubnets,
}
checkConf.Type = "bandwidth"
newCheckBytes, err := buildOneConfig(ver, checkConf, result)
Expect(err).NotTo(HaveOccurred())
args = &skel.CmdArgs{
ContainerID: "dummy3",
Netns: containerWithQoSNS.Path(),
IfName: containerWithQoSIFName,
StdinData: newCheckBytes,
}
err = testutils.CmdCheck(containerWithQoSNS.Path(), args.ContainerID, "", func() error { return cmdCheck(args) })
Expect(err).NotTo(HaveOccurred())
}
return nil
})).To(Succeed())
By("starting a tcp server on both containers")
portServerWithQoS, echoServerWithQoS = startEchoServerInNamespace(containerWithQoSNS)
portServerWithoutQoS, echoServerWithoutQoS = startEchoServerInNamespace(containerWithoutQoSNS)
})
AfterEach(func() {
Expect(os.RemoveAll(dataDir)).To(Succeed())
Expect(containerWithQoSNS.Close()).To(Succeed())
Expect(testutils.UnmountNS(containerWithQoSNS)).To(Succeed())
Expect(containerWithoutQoSNS.Close()).To(Succeed())
Expect(testutils.UnmountNS(containerWithoutQoSNS)).To(Succeed())
if echoServerWithoutQoS != nil {
echoServerWithoutQoS.Kill()
}
if echoServerWithQoS != nil {
echoServerWithQoS.Kill()
}
})
It("limits ingress traffic on veth device coming from included subnets", func() {
var runtimeWithLimit time.Duration
var runtimeWithoutLimit time.Duration
By("gather timing statistics about both containers")
By("sending tcp traffic to the container that has traffic shaped", func() {
start := time.Now()
result, err := types100.GetResult(containerWithQoSRes)
Expect(err).NotTo(HaveOccurred())
makeTCPClientInNS(hostNs.Path(), result.IPs[0].Address.IP.String(), portServerWithQoS, packetInBytes)
end := time.Now()
runtimeWithLimit = end.Sub(start)
log.Printf("Elapsed with qos %.2f", runtimeWithLimit.Seconds())
})
By("sending tcp traffic to the container that does not have traffic shaped", func() {
start := time.Now()
result, err := types100.GetResult(containerWithoutQoSRes)
Expect(err).NotTo(HaveOccurred())
makeTCPClientInNS(hostNs.Path(), result.IPs[0].Address.IP.String(), portServerWithoutQoS, packetInBytes)
end := time.Now()
runtimeWithoutLimit = end.Sub(start)
log.Printf("Elapsed without qos %.2f", runtimeWithoutLimit.Seconds())
})
Expect(runtimeWithLimit).To(BeNumerically(">", runtimeWithoutLimit+1000*time.Millisecond))
})
})
}
})

48
plugins/meta/bandwidth/bandwidth_suite_test.go
View File

@ -15,7 +15,6 @@ package main
import (
"bytes"
"encoding/json"
"fmt"
"io"
"net"
@ -31,11 +30,10 @@ import (
"github.com/onsi/gomega/gexec"
"github.com/vishvananda/netlink"
"github.com/containernetworking/cni/pkg/types"
"github.com/containernetworking/plugins/pkg/ns"
)
func TestHTB(t *testing.T) {
func TestTBF(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "plugins/meta/bandwidth")
}
@ -245,47 +243,3 @@ func createMacvlan(netNS ns.NetNS, master, macvlanName string) {
})
Expect(err).NotTo(HaveOccurred())
}
func buildOneConfig(cniVersion string, orig *PluginConf, prevResult types.Result) ([]byte, error) {
var err error
inject := map[string]interface{}{
"name": "myBWnet",
"cniVersion": cniVersion,
}
// Add previous plugin result
if prevResult != nil {
r, err := prevResult.GetAsVersion(cniVersion)
Expect(err).NotTo(HaveOccurred())
inject["prevResult"] = r
}
// Ensure every config uses the same name and version
config := make(map[string]interface{})
confBytes, err := json.Marshal(orig)
if err != nil {
return nil, err
}
err = json.Unmarshal(confBytes, &config)
if err != nil {
return nil, fmt.Errorf("unmarshal existing network bytes: %s", err)
}
for key, value := range inject {
config[key] = value
}
newBytes, err := json.Marshal(config)
if err != nil {
return nil, err
}
conf := &PluginConf{}
if err := json.Unmarshal(newBytes, &conf); err != nil {
return nil, fmt.Errorf("error parsing configuration: %s", err)
}
return newBytes, nil
}

261
plugins/meta/bandwidth/ifb_creator.go
View File

@ -15,8 +15,6 @@
package main
import (
"bytes"
"encoding/binary"
"fmt"
"net"
"syscall"
@ -26,24 +24,14 @@ import (
"github.com/containernetworking/plugins/pkg/ip"
)
const (
latencyInMillis = 25
UncappedRate uint64 = 100_000_000_000
ShapedClassMinorID uint16 = 48
UnShapedClassMinorID uint16 = 1
)
func CreateIfb(ifbDeviceName string, mtu int, qlen int) error {
if qlen < 1000 {
qlen = 1000
}
const latencyInMillis = 25
func CreateIfb(ifbDeviceName string, mtu int) error {
err := netlink.LinkAdd(&netlink.Ifb{
LinkAttrs: netlink.LinkAttrs{
Name: ifbDeviceName,
Flags: net.FlagUp,
MTU: mtu,
TxQLen: qlen,
Name: ifbDeviceName,
Flags: net.FlagUp,
MTU: mtu,
},
})
if err != nil {
@ -61,24 +49,15 @@ func TeardownIfb(deviceName string) error {
return err
}
func CreateIngressQdisc(rateInBits, burstInBits uint64, excludeSubnets []string, includeSubnets []string, hostDeviceName string) error {
func CreateIngressQdisc(rateInBits, burstInBits uint64, hostDeviceName string) error {
hostDevice, err := netlink.LinkByName(hostDeviceName)
if err != nil {
return fmt.Errorf("get host device: %s", err)
}
subnets := includeSubnets
exclude := false
if len(excludeSubnets) > 0 {
subnets = excludeSubnets
exclude = true
}
return createHTB(rateInBits, burstInBits, hostDevice.Attrs().Index, subnets, exclude)
return createTBF(rateInBits, burstInBits, hostDevice.Attrs().Index)
}
func CreateEgressQdisc(rateInBits, burstInBits uint64, excludeSubnets []string, includeSubnets []string, hostDeviceName string, ifbDeviceName string) error {
func CreateEgressQdisc(rateInBits, burstInBits uint64, hostDeviceName string, ifbDeviceName string) error {
ifbDevice, err := netlink.LinkByName(ifbDeviceName)
if err != nil {
return fmt.Errorf("get ifb device: %s", err)
@ -125,216 +104,44 @@ func CreateEgressQdisc(rateInBits, burstInBits uint64, excludeSubnets []string,
return fmt.Errorf("add filter: %s", err)
}
subnets := excludeSubnets
exclude := true
if len(includeSubnets) > 0 {
subnets = includeSubnets
exclude = false
}
// throttle traffic on ifb device
err = createHTB(rateInBits, burstInBits, ifbDevice.Attrs().Index, subnets, exclude)
err = createTBF(rateInBits, burstInBits, ifbDevice.Attrs().Index)
if err != nil {
// egress from the container/netns pov = ingress from the main netns/host pov
return fmt.Errorf("create htb container egress qos rules: %s", err)
return fmt.Errorf("create ifb qdisc: %s", err)
}
return nil
}
func createHTB(rateInBits, burstInBits uint64, linkIndex int, subnets []string, excludeSubnets bool) error {
// Netlink struct fields are not clear, let's use shell
defaultClassID := UnShapedClassMinorID
// If no subnets are specified, then shaping should apply to everything
if len(subnets) == 0 || excludeSubnets {
defaultClassID = ShapedClassMinorID
func createTBF(rateInBits, burstInBits uint64, linkIndex int) error {
// Equivalent to
// tc qdisc add dev link root tbf
// rate netConf.BandwidthLimits.Rate
// burst netConf.BandwidthLimits.Burst
if rateInBits <= 0 {
return fmt.Errorf("invalid rate: %d", rateInBits)
}
if burstInBits <= 0 {
return fmt.Errorf("invalid burst: %d", burstInBits)
}
rateInBytes := rateInBits / 8
burstInBytes := burstInBits / 8
bufferInBytes := buffer(rateInBytes, uint32(burstInBytes))
latency := latencyInUsec(latencyInMillis)
limitInBytes := limit(rateInBytes, latency, uint32(burstInBytes))
// Step 1 qdisc
// cmd := exec.Command("/usr/sbin/tc", "qdisc", "add", "dev", interfaceName, "root", "handle", "1:", "htb", "default", "30")
qdisc := &netlink.Htb{
qdisc := &netlink.Tbf{
QdiscAttrs: netlink.QdiscAttrs{
LinkIndex: linkIndex,
Handle: netlink.MakeHandle(1, 0),
Parent: netlink.HANDLE_ROOT,
},
Defcls: uint32(defaultClassID),
// No idea what these are so let's keep the default values from source code...
Version: 3,
Rate2Quantum: 10,
Limit: limitInBytes,
Rate: rateInBytes,
Buffer: bufferInBytes,
}
err := netlink.QdiscAdd(qdisc)
if err != nil {
return fmt.Errorf("error while creating qdisc: %s", err)
}
// Step 2 classes
rateInBytes := rateInBits / 8
burstInBytes := burstInBits / 8
bufferInBytes := buffer(rateInBytes, uint32(burstInBytes))
// The capped class for shaped traffic (included subnets or all but excluded subnets)
// cmd = exec.Command("/usr/sbin/tc", "class", "add", "dev", interfaceName, "parent", "1:", "classid", "1:30", "htb", "rate",
// fmt.Sprintf("%d", rateInBits), "burst", fmt.Sprintf("%d", burstInBits))
shapedClass := &netlink.HtbClass{
ClassAttrs: netlink.ClassAttrs{
LinkIndex: linkIndex,
Handle: netlink.MakeHandle(1, ShapedClassMinorID),
Parent: netlink.MakeHandle(1, 0),
},
Rate: rateInBytes,
Buffer: bufferInBytes,
// Let's set up the "burst" rate to twice the specified rate
Ceil: 2 * rateInBytes,
Cbuffer: bufferInBytes,
}
err = netlink.ClassAdd(shapedClass)
if err != nil {
return fmt.Errorf("error while creating htb default class: %s", err)
}
// The uncapped class for non shaped traffic (either all but included subnets or excluded subnets only)
// cmd = exec.Command("/usr/sbin/tc", "class", "add", "dev", interfaceName, "parent", "1:", "classid", "1:1", "htb",
// "rate", "100000000000")
bigRate := UncappedRate
unshapedClass := &netlink.HtbClass{
ClassAttrs: netlink.ClassAttrs{
LinkIndex: linkIndex,
Handle: netlink.MakeHandle(1, UnShapedClassMinorID),
Parent: qdisc.Handle,
},
Rate: bigRate,
Ceil: bigRate,
// No need for any burst, the minimum buffer size in q_htb.c should be enough to handle the rate which
// is already more than enough
}
err = netlink.ClassAdd(unshapedClass)
if err != nil {
return fmt.Errorf("error while creating htb uncapped class: %s", err)
}
// Now add filters to redirect subnets to the class 1 if excluded instead of the default one (30)
for _, subnet := range subnets {
// cmd = exec.Command("/usr/sbin/tc", "filter", "add", "dev", interfaceName, "parent", "1:", "protocol", protocol,
// "prio", "16", "u32", "match", "ip", "dst", subnet, "flowid", "1:1")
_, nw, err := net.ParseCIDR(subnet)
if err != nil {
return fmt.Errorf("bad subnet %s: %s", subnet, err)
}
var maskBytes []byte = nw.Mask
var subnetBytes []byte = nw.IP
if len(maskBytes) != len(subnetBytes) {
return fmt.Errorf("error using net lib for subnet %s len(maskBytes) != len(subnetBytes) "+
"(%d != %d) should not happen", subnet, len(maskBytes), len(subnetBytes))
}
isIpv4 := nw.IP.To4() != nil
protocol := syscall.ETH_P_IPV6
var prio uint16 = 15
var offset int32 = 24
keepBytes := 16
if isIpv4 {
protocol = syscall.ETH_P_IP
offset = 16
keepBytes = 4
// prio/pref needs to be changed if we change the protocol, looks like we cannot mix protocols with the same pref
prio = 16
}
if len(maskBytes) < keepBytes {
return fmt.Errorf("error with net lib, unexpected count of bytes for ipv4 mask (%d < %d)",
len(maskBytes), keepBytes)
}
if len(subnetBytes) < keepBytes {
return fmt.Errorf("error with net lib, unexpected count of bytes for ipv4 subnet (%d < %d)",
len(subnetBytes), keepBytes)
}
maskBytes = maskBytes[len(maskBytes)-keepBytes:]
subnetBytes = subnetBytes[len(subnetBytes)-keepBytes:]
// For ipv4 we should have at most 1 key, for ipv6 at most 4
keys := make([]netlink.TcU32Key, 0, 4)
for i := 0; i < len(maskBytes); i += 4 {
var mask, subnetI uint32
buf := bytes.NewReader(maskBytes[i : i+4])
err = binary.Read(buf, binary.BigEndian, &mask)
if err != nil {
return fmt.Errorf("error, htb filter, unable to build mask match filter, iter %d for subnet %s",
i, subnet)
}
if mask != 0 {
// If mask == 0, any value on this section will be a match and we do not need a filter for this
buf = bytes.NewReader(subnetBytes[i : i+4])
err = binary.Read(buf, binary.BigEndian, &subnetI)
if err != nil {
return fmt.Errorf("error, htb filter, unable to build subnet match filter, iter %d for subnet %s",
i, subnet)
}
keys = append(keys, netlink.TcU32Key{
Mask: mask,
Val: subnetI,
Off: offset,
OffMask: 0,
})
}
offset += 4
}
if len(keys) != cap(keys) {
shrinkedKeys := make([]netlink.TcU32Key, len(keys))
copied := copy(shrinkedKeys, keys)
if copied != len(keys) {
return fmt.Errorf("copy tc u32 keys error, for subnet %s copied %d != keys %d", subnet, copied, len(keys))
}
keys = shrinkedKeys
}
if isIpv4 && len(keys) > 1 {
return fmt.Errorf("error, htb ipv4 filter, unexpected rule length (%d > 1), for subnet %s",
len(keys), subnet)
} else if len(keys) > 4 {
return fmt.Errorf("error, htb ipv6 filter, unexpected rule length (%d > 4), for subnet %s",
len(keys), subnet)
}
// If len(keys) == 0, it means that we want to wildcard all traffic on the non default/uncapped class
var selector *netlink.TcU32Sel
if len(keys) > 0 {
selector = &netlink.TcU32Sel{
Nkeys: uint8(len(keys)),
Flags: netlink.TC_U32_TERMINAL,
Keys: keys,
}
}
classID := shapedClass.Handle
if excludeSubnets {
classID = unshapedClass.Handle
}
tcFilter := netlink.U32{
FilterAttrs: netlink.FilterAttrs{
LinkIndex: linkIndex,
Parent: qdisc.Handle,
Priority: prio,
Protocol: uint16(protocol),
},
ClassId: classID,
Sel: selector,
}
err = netlink.FilterAdd(&tcFilter)
if err != nil {
return fmt.Errorf("error, unable to create htb filter, details %s", err)
}
return fmt.Errorf("create qdisc: %s", err)
}
return nil
}
@ -346,3 +153,11 @@ func time2Tick(time uint32) uint32 {
func buffer(rate uint64, burst uint32) uint32 {
return time2Tick(uint32(float64(burst) * float64(netlink.TIME_UNITS_PER_SEC) / float64(rate)))
}
func limit(rate uint64, latency float64, buffer uint32) uint32 {
return uint32(float64(rate)*latency/float64(netlink.TIME_UNITS_PER_SEC)) + buffer
}
func latencyInUsec(latencyInMillis float64) float64 {
return float64(netlink.TIME_UNITS_PER_SEC) * (latencyInMillis / 1000.0)
}

184
plugins/meta/bandwidth/main.go
View File

@ -18,7 +18,6 @@ import (
"encoding/json"
"fmt"
"math"
"net"
"github.com/vishvananda/netlink"
@ -40,12 +39,11 @@ const (
// BandwidthEntry corresponds to a single entry in the bandwidth argument,
// see CONVENTIONS.md
type BandwidthEntry struct {
UnshapedSubnets []string `json:"unshapedSubnets"` // Ipv4/ipv6 subnets to be excluded from traffic shaping. UnshapedSubnets and ShapedSubnets parameters are mutually exlusive
ShapedSubnets []string `json:"shapedSubnets"` // Ipv4/ipv6 subnets to be included in traffic shaping. UnshapedSubnets and ShapedSubnets parameters are mutually exlusive
IngressRate uint64 `json:"ingressRate"` // Bandwidth rate in bps for traffic through container. 0 for no limit. If ingressRate is set, ingressBurst must also be set
IngressBurst uint64 `json:"ingressBurst"` // Bandwidth burst in bits for traffic through container. 0 for no limit. If ingressBurst is set, ingressRate must also be set
EgressRate uint64 `json:"egressRate"` // Bandwidth rate in bps for traffic through container. 0 for no limit. If egressRate is set, egressBurst must also be set
EgressBurst uint64 `json:"egressBurst"` // Bandwidth burst in bits for traffic through container. 0 for no limit. If egressBurst is set, egressRate must also be set
IngressRate uint64 `json:"ingressRate"` // Bandwidth rate in bps for traffic through container. 0 for no limit. If ingressRate is set, ingressBurst must also be set
IngressBurst uint64 `json:"ingressBurst"` // Bandwidth burst in bits for traffic through container. 0 for no limit. If ingressBurst is set, ingressRate must also be set
EgressRate uint64 `json:"egressRate"` // Bandwidth rate in bps for traffic through container. 0 for no limit. If egressRate is set, egressBurst must also be set
EgressBurst uint64 `json:"egressBurst"` // Bandwidth burst in bits for traffic through container. 0 for no limit. If egressBurst is set, egressRate must also be set
}
func (bw *BandwidthEntry) isZero() bool {
@ -98,21 +96,10 @@ func parseConfig(stdin []byte) (*PluginConf, error) {
}
func getBandwidth(conf *PluginConf) *BandwidthEntry {
bw := conf.BandwidthEntry
if bw == nil && conf.RuntimeConfig.Bandwidth != nil {
bw = conf.RuntimeConfig.Bandwidth
if conf.BandwidthEntry == nil && conf.RuntimeConfig.Bandwidth != nil {
return conf.RuntimeConfig.Bandwidth
}
if bw != nil {
if bw.UnshapedSubnets == nil {
bw.UnshapedSubnets = make([]string, 0)
}
if bw.ShapedSubnets == nil {
bw.ShapedSubnets = make([]string, 0)
}
}
return bw
return conf.BandwidthEntry
}
func validateRateAndBurst(rate, burst uint64) error {
@ -132,13 +119,13 @@ func getIfbDeviceName(networkName string, containerID string) string {
return utils.MustFormatHashWithPrefix(maxIfbDeviceLength, ifbDevicePrefix, networkName+containerID)
}
func getMTUAndQLen(deviceName string) (int, int, error) {
func getMTU(deviceName string) (int, error) {
link, err := netlink.LinkByName(deviceName)
if err != nil {
return -1, -1, err
return -1, err
}
return link.Attrs().MTU, link.Attrs().TxQLen, nil
return link.Attrs().MTU, nil
}
// get the veth peer of container interface in host namespace
@ -172,28 +159,6 @@ func getHostInterface(interfaces []*current.Interface, containerIfName string, n
return nil, fmt.Errorf("no veth peer of container interface found in host ns")
}
func validateSubnets(unshapedSubnets []string, shapedSubnets []string) error {
if len(unshapedSubnets) > 0 && len(shapedSubnets) > 0 {
return fmt.Errorf("unshapedSubnets and shapedSubnets cannot be both specified, one of them should be discarded")
}
for _, subnet := range unshapedSubnets {
_, _, err := net.ParseCIDR(subnet)
if err != nil {
return fmt.Errorf("bad subnet %q provided, details %s", subnet, err)
}
}
for _, subnet := range shapedSubnets {
_, _, err := net.ParseCIDR(subnet)
if err != nil {
return fmt.Errorf("bad subnet %q provided, details %s", subnet, err)
}
}
return nil
}
func cmdAdd(args *skel.CmdArgs) error {
conf, err := parseConfig(args.StdinData)
if err != nil {
@ -205,10 +170,6 @@ func cmdAdd(args *skel.CmdArgs) error {
return types.PrintResult(conf.PrevResult, conf.CNIVersion)
}
if err = validateSubnets(bandwidth.UnshapedSubnets, bandwidth.ShapedSubnets); err != nil {
return err
}
if conf.PrevResult == nil {
return fmt.Errorf("must be called as chained plugin")
}
@ -230,22 +191,21 @@ func cmdAdd(args *skel.CmdArgs) error {
}
if bandwidth.IngressRate > 0 && bandwidth.IngressBurst > 0 {
err = CreateIngressQdisc(bandwidth.IngressRate, bandwidth.IngressBurst,
bandwidth.UnshapedSubnets, bandwidth.ShapedSubnets, hostInterface.Name)
err = CreateIngressQdisc(bandwidth.IngressRate, bandwidth.IngressBurst, hostInterface.Name)
if err != nil {
return err
}
}
if bandwidth.EgressRate > 0 && bandwidth.EgressBurst > 0 {
mtu, qlen, err := getMTUAndQLen(hostInterface.Name)
mtu, err := getMTU(hostInterface.Name)
if err != nil {
return err
}
ifbDeviceName := getIfbDeviceName(conf.Name, args.ContainerID)
err = CreateIfb(ifbDeviceName, mtu, qlen)
err = CreateIfb(ifbDeviceName, mtu)
if err != nil {
return err
}
@ -259,9 +219,7 @@ func cmdAdd(args *skel.CmdArgs) error {
Name: ifbDeviceName,
Mac: ifbDevice.Attrs().HardwareAddr.String(),
})
err = CreateEgressQdisc(bandwidth.EgressRate, bandwidth.EgressBurst,
bandwidth.UnshapedSubnets, bandwidth.ShapedSubnets, hostInterface.Name,
ifbDeviceName)
err = CreateEgressQdisc(bandwidth.EgressRate, bandwidth.EgressBurst, hostInterface.Name, ifbDeviceName)
if err != nil {
return err
}
@ -340,99 +298,75 @@ func cmdCheck(args *skel.CmdArgs) error {
bandwidth := getBandwidth(bwConf)
if err = validateSubnets(bandwidth.UnshapedSubnets, bandwidth.ShapedSubnets); err != nil {
return fmt.Errorf("failed to check subnets, details %s", err)
}
if bandwidth.IngressRate > 0 && bandwidth.IngressBurst > 0 {
rateInBytes := bandwidth.IngressRate / 8
burstInBytes := bandwidth.IngressBurst / 8
bufferInBytes := buffer(rateInBytes, uint32(burstInBytes))
err = checkHTB(link, rateInBytes, bufferInBytes, bandwidth.ShapedSubnets)
latency := latencyInUsec(latencyInMillis)
limitInBytes := limit(rateInBytes, latency, uint32(burstInBytes))
qdiscs, err := SafeQdiscList(link)
if err != nil {
return err
}
if len(qdiscs) == 0 {
return fmt.Errorf("Failed to find qdisc")
}
for _, qdisc := range qdiscs {
tbf, isTbf := qdisc.(*netlink.Tbf)
if !isTbf {
break
}
if tbf.Rate != rateInBytes {
return fmt.Errorf("Rate doesn't match")
}
if tbf.Limit != limitInBytes {
return fmt.Errorf("Limit doesn't match")
}
if tbf.Buffer != bufferInBytes {
return fmt.Errorf("Buffer doesn't match")
}
}
}
if bandwidth.EgressRate > 0 && bandwidth.EgressBurst > 0 {
rateInBytes := bandwidth.EgressRate / 8
burstInBytes := bandwidth.EgressBurst / 8
bufferInBytes := buffer(rateInBytes, uint32(burstInBytes))
latency := latencyInUsec(latencyInMillis)
limitInBytes := limit(rateInBytes, latency, uint32(burstInBytes))
ifbDeviceName := getIfbDeviceName(bwConf.Name, args.ContainerID)
ifbDevice, err := netlink.LinkByName(ifbDeviceName)
if err != nil {
return fmt.Errorf("get ifb device: %s", err)
}
err = checkHTB(ifbDevice, rateInBytes, bufferInBytes, bandwidth.ShapedSubnets)
qdiscs, err := SafeQdiscList(ifbDevice)
if err != nil {
return err
}
}
return nil
}
func checkHTB(link netlink.Link, rateInBytes uint64, bufferInBytes uint32, shapedSubnets []string) error {
qdiscs, err := SafeQdiscList(link)
if err != nil {
return err
}
if len(qdiscs) == 0 {
return fmt.Errorf("Failed to find qdisc")
}
foundHTB := false
for _, qdisc := range qdiscs {
htb, isHtb := qdisc.(*netlink.Htb)
if !isHtb {
continue
if len(qdiscs) == 0 {
return fmt.Errorf("Failed to find qdisc")
}
if foundHTB {
return fmt.Errorf("Several htb qdisc found for device %s", link.Attrs().Name)
}
foundHTB = true
defaultClassMinorID := ShapedClassMinorID
if len(shapedSubnets) > 0 {
defaultClassMinorID = UnShapedClassMinorID
}
if htb.Defcls != uint32(defaultClassMinorID) {
return fmt.Errorf("Default class does not match")
}
classes, err := netlink.ClassList(link, htb.Handle)
if err != nil {
return fmt.Errorf("Unable to list classes bound to htb qdisc for device %s. Details %s",
link.Attrs().Name, err)
}
if len(classes) != 2 {
return fmt.Errorf("Number of htb classes does not match for device %s (%d != 2)",
link.Attrs().Name, len(classes))
}
for _, c := range classes {
htbClass, isHtb := c.(*netlink.HtbClass)
if !isHtb {
return fmt.Errorf("Unexpected class for parent htb qdisc bound to device %s", link.Attrs().Name)
for _, qdisc := range qdiscs {
tbf, isTbf := qdisc.(*netlink.Tbf)
if !isTbf {
break
}
if htbClass.Handle == htb.Defcls {
if htbClass.Rate != rateInBytes {
return fmt.Errorf("Rate does not match for the default class for device %s (%d != %d)",
link.Attrs().Name, htbClass.Rate, rateInBytes)
}
if htbClass.Buffer != bufferInBytes {
return fmt.Errorf("Burst buffer size does not match for the default class for device %s (%d != %d)",
link.Attrs().Name, htbClass.Buffer, bufferInBytes)
}
} else if htbClass.Handle == netlink.MakeHandle(1, 1) {
if htbClass.Rate != UncappedRate {
return fmt.Errorf("Rate does not match for the uncapped class for device %s (%d != %d)",
link.Attrs().Name, htbClass.Rate, UncappedRate)
}
if tbf.Rate != rateInBytes {
return fmt.Errorf("Rate doesn't match")
}
if tbf.Limit != limitInBytes {
return fmt.Errorf("Limit doesn't match")
}
if tbf.Buffer != bufferInBytes {
return fmt.Errorf("Buffer doesn't match")
}
}
// TODO: check subnet filters
}
return nil