ITQC/containernetworking-plugins
24
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 34 Wiki Activity
containernetworking-plugins/plugins/meta/bandwidth/bandwidth_linux_test.go
Raphael ab0b386b4e bandwidth: possibility to specify shaped subnets or to exclude some from shaping 
Signed-off-by: Raphael <oOraph@users.noreply.github.com>
2024-04-08 15:39:47 +09:00

2918 lines
102 KiB
Go
Raw Blame History

// Copyright 2018 CNI authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import (
"context"
"encoding/json"
"fmt"
"log"
"math"
"net"
"os"
"syscall"
"time"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/onsi/gomega/gexec"
"github.com/vishvananda/netlink"
"github.com/containernetworking/cni/pkg/invoke"
"github.com/containernetworking/cni/pkg/skel"
"github.com/containernetworking/cni/pkg/types"
types100 "github.com/containernetworking/cni/pkg/types/100"
"github.com/containernetworking/plugins/pkg/ns"
"github.com/containernetworking/plugins/pkg/testutils"
)
func buildOneConfig(cniVersion string, orig *PluginConf, prevResult types.Result) ([]byte, error) {
var err error
name := "myBWnet"
inject := map[string]interface{}{
"name": name,
"cniVersion": cniVersion,
}
// Add previous plugin result
if prevResult != nil {
r, err := prevResult.GetAsVersion(cniVersion)
Expect(err).NotTo(HaveOccurred())
inject["prevResult"] = r
}
// Ensure every config uses the same name and version
config := make(map[string]interface{})
confBytes, err := json.Marshal(orig)
if err != nil {
return nil, err
}
err = json.Unmarshal(confBytes, &config)
if err != nil {
return nil, fmt.Errorf("unmarshal existing network bytes: %s", err)
}
for key, value := range inject {
config[key] = value
}
newBytes, err := json.Marshal(config)
if err != nil {
return nil, err
}
conf := &PluginConf{}
if err := json.Unmarshal(newBytes, &conf); err != nil {
return nil, fmt.Errorf("error parsing configuration: %s", err)
}
return newBytes, nil
}
var _ = Describe("bandwidth test", func() {
var (
hostNs ns.NetNS
containerNs ns.NetNS
ifbDeviceName string
hostIfname string
containerIfname string
hostIP net.IP
containerIP net.IP
hostIfaceMTU int
)
BeforeEach(func() {
var err error
hostIfname = "host-veth"
containerIfname = "container-veth"
hostNs, err = testutils.NewNS()
Expect(err).NotTo(HaveOccurred())
containerNs, err = testutils.NewNS()
Expect(err).NotTo(HaveOccurred())
hostIP = net.IP{169, 254, 0, 1}
containerIP = net.IP{10, 254, 0, 1}
hostIfaceMTU = 1024
ifbDeviceName = "bwpa8eda89404b7"
createVeth(hostNs, hostIfname, containerNs, containerIfname, hostIP, containerIP, hostIfaceMTU)
})
AfterEach(func() {
Expect(containerNs.Close()).To(Succeed())
Expect(testutils.UnmountNS(containerNs)).To(Succeed())
Expect(hostNs.Close()).To(Succeed())
Expect(testutils.UnmountNS(hostNs)).To(Succeed())
})
// Bandwidth requires host-side interface info, and thus only
// supports 0.3.0 and later CNI versions
for _, ver := range []string{"0.3.0", "0.3.1", "0.4.0", "1.0.0"} {
// Redefine ver inside for scope so real value is picked up by each dynamically defined It()
// See Gingkgo's "Patterns for dynamically generating tests" documentation.
ver := ver
Describe("cmdADD", func() {
It(fmt.Sprintf("[%s] works with a Veth pair without any unbounded traffic", ver), func() {
conf := fmt.Sprintf(`{
"cniVersion": "%s",
"name": "cni-plugin-bandwidth-test",
"type": "bandwidth",
"ingressRate": 8,
"ingressBurst": 8,
"egressRate": 16,
"egressBurst": 12,
"prevResult": {
"interfaces": [
{
"name": "%s",
"sandbox": ""
},
{
"name": "%s",
"sandbox": "%s"
}
],
"ips": [
{
"version": "4",
"address": "%s/24",
"gateway": "10.0.0.1",
"interface": 1
}
],
"routes": []
}
}`, ver, hostIfname, containerIfname, containerNs.Path(), containerIP.String())
args := &skel.CmdArgs{
ContainerID: "dummy",
Netns: containerNs.Path(),
IfName: containerIfname,
StdinData: []byte(conf),
}
// Container egress (host ingress)
Expect(hostNs.Do(func(netNS ns.NetNS) error {
defer GinkgoRecover()
r, out, err := testutils.CmdAdd(containerNs.Path(), args.ContainerID, "", []byte(conf), func() error { return cmdAdd(args) })
Expect(err).NotTo(HaveOccurred(), string(out))
result, err := types100.GetResult(r)
Expect(err).NotTo(HaveOccurred())
Expect(result.Interfaces).To(HaveLen(3))
Expect(result.Interfaces[2].Name).To(Equal(ifbDeviceName))
Expect(result.Interfaces[2].Sandbox).To(Equal(""))
ifbLink, err := netlink.LinkByName(ifbDeviceName)
Expect(err).NotTo(HaveOccurred())
Expect(ifbLink.Attrs().MTU).To(Equal(hostIfaceMTU))
qdiscs, err := netlink.QdiscList(ifbLink)
Expect(err).NotTo(HaveOccurred())
Expect(qdiscs).To(HaveLen(1))
Expect(qdiscs[0].Attrs().LinkIndex).To(Equal(ifbLink.Attrs().Index))
Expect(qdiscs[0]).To(BeAssignableToTypeOf(&netlink.Htb{}))
Expect(qdiscs[0].(*netlink.Htb).Defcls).To(Equal(uint32(ShapedClassMinorID)))
classes, err := netlink.ClassList(ifbLink, qdiscs[0].Attrs().Handle)
Expect(err).NotTo(HaveOccurred())
Expect(classes).To(HaveLen(2))
// Uncapped class
Expect(classes[0]).To(BeAssignableToTypeOf(&netlink.HtbClass{}))
Expect(classes[0].(*netlink.HtbClass).Handle).To(Equal(netlink.MakeHandle(1, 1)))
Expect(classes[0].(*netlink.HtbClass).Rate).To(Equal(UncappedRate))
Expect(classes[0].(*netlink.HtbClass).Buffer).To(Equal(uint32(0)))
Expect(classes[0].(*netlink.HtbClass).Ceil).To(Equal(UncappedRate))
Expect(classes[0].(*netlink.HtbClass).Cbuffer).To(Equal(uint32(0)))
// Class with traffic shapping settings
Expect(classes[1]).To(BeAssignableToTypeOf(&netlink.HtbClass{}))
Expect(classes[1].(*netlink.HtbClass).Handle).To(Equal(netlink.MakeHandle(1, uint16(qdiscs[0].(*netlink.Htb).Defcls))))
Expect(classes[1].(*netlink.HtbClass).Rate).To(Equal(uint64(2)))
// Expect(classes[1].(*netlink.HtbClass).Buffer).To(Equal(uint32(7812500)))
Expect(classes[1].(*netlink.HtbClass).Ceil).To(Equal(uint64(4)))
// Expect(classes[1].(*netlink.HtbClass).Cbuffer).To(Equal(uint32(0)))
// Since we do not exclude anything from egress traffic shapping, we should not find any filter
filters, err := netlink.FilterList(ifbLink, qdiscs[0].Attrs().Handle)
Expect(err).NotTo(HaveOccurred())
Expect(filters).To(BeEmpty())
hostVethLink, err := netlink.LinkByName(hostIfname)
Expect(err).NotTo(HaveOccurred())
qdiscFilters, err := netlink.FilterList(hostVethLink, netlink.MakeHandle(0xffff, 0))
Expect(err).NotTo(HaveOccurred())
Expect(qdiscFilters).To(HaveLen(1))
Expect(qdiscFilters[0].(*netlink.U32).Actions[0].(*netlink.MirredAction).Ifindex).To(Equal(ifbLink.Attrs().Index))
return nil
})).To(Succeed())
// Container ingress (host egress)
Expect(hostNs.Do(func(n ns.NetNS) error {
defer GinkgoRecover()
vethLink, err := netlink.LinkByName(hostIfname)
Expect(err).NotTo(HaveOccurred())
qdiscs, err := netlink.QdiscList(vethLink)
Expect(err).NotTo(HaveOccurred())
Expect(qdiscs).To(HaveLen(2))
Expect(qdiscs[0].Attrs().LinkIndex).To(Equal(vethLink.Attrs().Index))
Expect(qdiscs[0]).To(BeAssignableToTypeOf(&netlink.Htb{}))
Expect(qdiscs[0].(*netlink.Htb).Defcls).To(Equal(uint32(ShapedClassMinorID)))
classes, err := netlink.ClassList(vethLink, qdiscs[0].Attrs().Handle)
Expect(err).NotTo(HaveOccurred())
Expect(classes).To(HaveLen(2))
// Uncapped class
Expect(classes[0]).To(BeAssignableToTypeOf(&netlink.HtbClass{}))
Expect(classes[0].(*netlink.HtbClass).Handle).To(Equal(netlink.MakeHandle(1, 1)))
Expect(classes[0].(*netlink.HtbClass).Rate).To(Equal(UncappedRate))
Expect(classes[0].(*netlink.HtbClass).Buffer).To(Equal(uint32(0)))
Expect(classes[0].(*netlink.HtbClass).Ceil).To(Equal(UncappedRate))
Expect(classes[0].(*netlink.HtbClass).Cbuffer).To(Equal(uint32(0)))
// Class with traffic shapping settings
Expect(classes[1]).To(BeAssignableToTypeOf(&netlink.HtbClass{}))
Expect(classes[1].(*netlink.HtbClass).Handle).To(Equal(netlink.MakeHandle(1, uint16(qdiscs[0].(*netlink.Htb).Defcls))))
Expect(classes[1].(*netlink.HtbClass).Rate).To(Equal(uint64(1)))
// Expect(classes[1].(*netlink.HtbClass).Buffer).To(Equal(uint32(15625000)))
Expect(classes[1].(*netlink.HtbClass).Ceil).To(Equal(uint64(2)))
// Expect(classes[1].(*netlink.HtbClass).Cbuffer).To(Equal(uint32(0)))
// Since we do not exclude anything from ingress traffic shapping, we should not find any filter
filters, err := netlink.FilterList(vethLink, qdiscs[0].Attrs().Handle)
Expect(err).NotTo(HaveOccurred())
Expect(filters).To(BeEmpty())
return nil
})).To(Succeed())
})
It(fmt.Sprintf("[%s] works with a Veth pair with some ipv4 and ipv6 unbounded traffic", ver), func() {
conf := fmt.Sprintf(`{
"cniVersion": "%s",
"name": "cni-plugin-bandwidth-test",
"type": "bandwidth",
"ingressRate": 8,
"ingressBurst": 8,
"egressRate": 16,
"egressBurst": 12,
"unshapedSubnets": [
"10.0.0.0/8",
"fd00:db8:abcd:1234:e000::/68"
],
"prevResult": {
"interfaces": [
{
"name": "%s",
"sandbox": ""
},
{
"name": "%s",
"sandbox": "%s"
}
],
"ips": [
{
"version": "4",
"address": "%s/24",
"gateway": "10.0.0.1",
"interface": 1
}
],
"routes": []
}
}`, ver, hostIfname, containerIfname, containerNs.Path(), containerIP.String())
args := &skel.CmdArgs{
ContainerID: "dummy",
Netns: containerNs.Path(),
IfName: containerIfname,
StdinData: []byte(conf),
}
// Container egress (host ingress)
Expect(hostNs.Do(func(netNS ns.NetNS) error {
defer GinkgoRecover()
r, out, err := testutils.CmdAdd(containerNs.Path(), args.ContainerID, "", []byte(conf), func() error { return cmdAdd(args) })
Expect(err).NotTo(HaveOccurred(), string(out))
result, err := types100.GetResult(r)
Expect(err).NotTo(HaveOccurred())
Expect(result.Interfaces).To(HaveLen(3))
Expect(result.Interfaces[2].Name).To(Equal(ifbDeviceName))
Expect(result.Interfaces[2].Sandbox).To(Equal(""))
ifbLink, err := netlink.LinkByName(ifbDeviceName)
Expect(err).NotTo(HaveOccurred())
Expect(ifbLink.Attrs().MTU).To(Equal(hostIfaceMTU))
qdiscs, err := netlink.QdiscList(ifbLink)
Expect(err).NotTo(HaveOccurred())
Expect(qdiscs).To(HaveLen(1))
Expect(qdiscs[0].Attrs().LinkIndex).To(Equal(ifbLink.Attrs().Index))
Expect(qdiscs[0]).To(BeAssignableToTypeOf(&netlink.Htb{}))
Expect(qdiscs[0].(*netlink.Htb).Defcls).To(Equal(uint32(ShapedClassMinorID)))
classes, err := netlink.ClassList(ifbLink, qdiscs[0].Attrs().Handle)
Expect(err).NotTo(HaveOccurred())
Expect(classes).To(HaveLen(2))
// Uncapped class
Expect(classes[0]).To(BeAssignableToTypeOf(&netlink.HtbClass{}))
Expect(classes[0].(*netlink.HtbClass).Handle).To(Equal(netlink.MakeHandle(1, 1)))
Expect(classes[0].(*netlink.HtbClass).Rate).To(Equal(UncappedRate))
Expect(classes[0].(*netlink.HtbClass).Buffer).To(Equal(uint32(0)))
Expect(classes[0].(*netlink.HtbClass).Ceil).To(Equal(UncappedRate))
Expect(classes[0].(*netlink.HtbClass).Cbuffer).To(Equal(uint32(0)))
// Class with traffic shapping settings
Expect(classes[1]).To(BeAssignableToTypeOf(&netlink.HtbClass{}))
Expect(classes[1].(*netlink.HtbClass).Handle).To(Equal(netlink.MakeHandle(1, uint16(qdiscs[0].(*netlink.Htb).Defcls))))
Expect(classes[1].(*netlink.HtbClass).Rate).To(Equal(uint64(2)))
// Expect(classes[1].(*netlink.HtbClass).Buffer).To(Equal(uint32(7812500)))
Expect(classes[1].(*netlink.HtbClass).Ceil).To(Equal(uint64(4)))
// Expect(classes[1].(*netlink.HtbClass).Cbuffer).To(Equal(uint32(0)))
filters, err := netlink.FilterList(ifbLink, qdiscs[0].Attrs().Handle)
Expect(err).NotTo(HaveOccurred())
Expect(filters).To(HaveLen(2))
// traffic to fd00:db8:abcd:1234:e000::/68 redirected to uncapped class
Expect(filters[0]).To(BeAssignableToTypeOf(&netlink.U32{}))
Expect(filters[0].(*netlink.U32).Actions).To(BeEmpty())
Expect(filters[0].Attrs().Protocol).To(Equal(uint16(syscall.ETH_P_IPV6)))
Expect(filters[0].Attrs().LinkIndex).To(Equal(ifbLink.Attrs().Index))
Expect(filters[0].Attrs().Priority).To(Equal(uint16(15)))
Expect(filters[0].Attrs().Parent).To(Equal(qdiscs[0].Attrs().Handle))
Expect(filters[0].(*netlink.U32).ClassId).To(Equal(netlink.MakeHandle(1, 1)))
filterSel := filters[0].(*netlink.U32).Sel
Expect(filterSel).To(BeAssignableToTypeOf(&netlink.TcU32Sel{}))
Expect(filterSel.Flags).To(Equal(uint8(netlink.TC_U32_TERMINAL)))
Expect(filterSel.Keys).To(HaveLen(3))
Expect(filterSel.Nkeys).To(Equal(uint8(3)))
// The filter should match to fd00:db8:abcd:1234:e000::/68 dst address in other words it should be:
// match 0xfd000db8/0xffffffff at 24
// match 0xabcd1234/0xffffffff at 28
// match 0xe0000000/0xf0000000 at 32
// (and last match discarded because it would be equivalent to a matchall/true condition at 36)
Expect(filterSel.Keys[0].Off).To(Equal(int32(24)))
Expect(filterSel.Keys[0].Val).To(Equal(uint32(4244639160)))
Expect(filterSel.Keys[0].Mask).To(Equal(uint32(4294967295)))
Expect(filterSel.Keys[1].Off).To(Equal(int32(28)))
Expect(filterSel.Keys[1].Val).To(Equal(uint32(2882343476)))
Expect(filterSel.Keys[1].Mask).To(Equal(uint32(4294967295)))
Expect(filterSel.Keys[2].Off).To(Equal(int32(32)))
Expect(filterSel.Keys[2].Val).To(Equal(uint32(3758096384)))
Expect(filterSel.Keys[2].Mask).To(Equal(uint32(4026531840)))
// traffic to 10.0.0.0/8 redirected to uncapped class
Expect(filters[1]).To(BeAssignableToTypeOf(&netlink.U32{}))
Expect(filters[1].(*netlink.U32).Actions).To(BeEmpty())
Expect(filters[1].Attrs().Protocol).To(Equal(uint16(syscall.ETH_P_IP)))
Expect(filters[1].Attrs().LinkIndex).To(Equal(ifbLink.Attrs().Index))
Expect(filters[1].Attrs().Priority).To(Equal(uint16(16)))
Expect(filters[1].Attrs().Parent).To(Equal(qdiscs[0].Attrs().Handle))
Expect(filters[1].(*netlink.U32).ClassId).To(Equal(netlink.MakeHandle(1, 1)))
filterSel = filters[1].(*netlink.U32).Sel
Expect(filterSel).To(BeAssignableToTypeOf(&netlink.TcU32Sel{}))
Expect(filterSel.Flags).To(Equal(uint8(netlink.TC_U32_TERMINAL)))
Expect(filterSel.Keys).To(HaveLen(1))
Expect(filterSel.Nkeys).To(Equal(uint8(1)))
// The filter should match to 10.0.0.0/8 dst address in other words it should be:
// match 0a000000/ff000000 at 16
selKey := filterSel.Keys[0]
Expect(selKey.Val).To(Equal(uint32(10 * math.Pow(256, 3))))
Expect(selKey.Off).To(Equal(int32(16)))
Expect(selKey.Mask).To(Equal(uint32(255 * math.Pow(256, 3))))
hostVethLink, err := netlink.LinkByName(hostIfname)
Expect(err).NotTo(HaveOccurred())
qdiscFilters, err := netlink.FilterList(hostVethLink, netlink.MakeHandle(0xffff, 0))
Expect(err).NotTo(HaveOccurred())
Expect(qdiscFilters).To(HaveLen(1))
Expect(qdiscFilters[0].(*netlink.U32).Actions[0].(*netlink.MirredAction).Ifindex).To(Equal(ifbLink.Attrs().Index))
return nil
})).To(Succeed())
// Container ingress (host egress)
Expect(hostNs.Do(func(n ns.NetNS) error {
defer GinkgoRecover()
vethLink, err := netlink.LinkByName(hostIfname)
Expect(err).NotTo(HaveOccurred())
qdiscs, err := netlink.QdiscList(vethLink)
Expect(err).NotTo(HaveOccurred())
Expect(qdiscs).To(HaveLen(2))
Expect(qdiscs[0].Attrs().LinkIndex).To(Equal(vethLink.Attrs().Index))
Expect(qdiscs[0]).To(BeAssignableToTypeOf(&netlink.Htb{}))
Expect(qdiscs[0].(*netlink.Htb).Defcls).To(Equal(uint32(ShapedClassMinorID)))
classes, err := netlink.ClassList(vethLink, qdiscs[0].Attrs().Handle)
Expect(err).NotTo(HaveOccurred())
Expect(classes).To(HaveLen(2))
// Uncapped class
Expect(classes[0]).To(BeAssignableToTypeOf(&netlink.HtbClass{}))
Expect(classes[0].(*netlink.HtbClass).Handle).To(Equal(netlink.MakeHandle(1, 1)))
Expect(classes[0].(*netlink.HtbClass).Rate).To(Equal(UncappedRate))
Expect(classes[0].(*netlink.HtbClass).Buffer).To(Equal(uint32(0)))
Expect(classes[0].(*netlink.HtbClass).Ceil).To(Equal(UncappedRate))
Expect(classes[0].(*netlink.HtbClass).Cbuffer).To(Equal(uint32(0)))
// Class with traffic shapping settings
Expect(classes[1]).To(BeAssignableToTypeOf(&netlink.HtbClass{}))
Expect(classes[1].(*netlink.HtbClass).Handle).To(Equal(netlink.MakeHandle(1, uint16(qdiscs[0].(*netlink.Htb).Defcls))))
Expect(classes[1].(*netlink.HtbClass).Rate).To(Equal(uint64(1)))
// Expect(classes[1].(*netlink.HtbClass).Buffer).To(Equal(uint32(15625000)))
Expect(classes[1].(*netlink.HtbClass).Ceil).To(Equal(uint64(2)))
// Expect(classes[1].(*netlink.HtbClass).Cbuffer).To(Equal(uint32(0)))
filters, err := netlink.FilterList(vethLink, qdiscs[0].Attrs().Handle)
Expect(err).NotTo(HaveOccurred())
Expect(filters).To(HaveLen(2))
// traffic to fd00:db8:abcd:1234:e000::/68 redirected to uncapped class
Expect(filters[0]).To(BeAssignableToTypeOf(&netlink.U32{}))
Expect(filters[0].(*netlink.U32).Actions).To(BeEmpty())
Expect(filters[0].Attrs().Protocol).To(Equal(uint16(syscall.ETH_P_IPV6)))
Expect(filters[0].Attrs().LinkIndex).To(Equal(vethLink.Attrs().Index))
Expect(filters[0].Attrs().Priority).To(Equal(uint16(15)))
Expect(filters[0].Attrs().Parent).To(Equal(qdiscs[0].Attrs().Handle))
Expect(filters[0].(*netlink.U32).ClassId).To(Equal(netlink.MakeHandle(1, 1)))
filterSel := filters[0].(*netlink.U32).Sel
Expect(filterSel).To(BeAssignableToTypeOf(&netlink.TcU32Sel{}))
Expect(filterSel.Flags).To(Equal(uint8(netlink.TC_U32_TERMINAL)))
Expect(filterSel.Keys).To(HaveLen(3))
Expect(filterSel.Nkeys).To(Equal(uint8(3)))
// The filter should match to fd00:db8:abcd:1234:e000::/68 dst address in other words it should be:
// match 0xfd000db8/0xffffffff at 24
// match 0xabcd1234/0xffffffff at 28
// match 0xe0000000/0xf0000000 at 32
// (and last match discarded because it would be equivalent to a matchall/true condition at 36)
Expect(filterSel.Keys[0].Off).To(Equal(int32(24)))
Expect(filterSel.Keys[0].Val).To(Equal(uint32(4244639160)))
Expect(filterSel.Keys[0].Mask).To(Equal(uint32(4294967295)))
Expect(filterSel.Keys[1].Off).To(Equal(int32(28)))
Expect(filterSel.Keys[1].Val).To(Equal(uint32(2882343476)))
Expect(filterSel.Keys[1].Mask).To(Equal(uint32(4294967295)))
Expect(filterSel.Keys[2].Off).To(Equal(int32(32)))
Expect(filterSel.Keys[2].Val).To(Equal(uint32(3758096384)))
Expect(filterSel.Keys[2].Mask).To(Equal(uint32(4026531840)))
// traffic to 10.0.0.0/8 redirected to uncapped class
Expect(filters[1]).To(BeAssignableToTypeOf(&netlink.U32{}))
Expect(filters[1].(*netlink.U32).Actions).To(BeEmpty())
Expect(filters[1].Attrs().Protocol).To(Equal(uint16(syscall.ETH_P_IP)))
Expect(filters[1].Attrs().LinkIndex).To(Equal(vethLink.Attrs().Index))
Expect(filters[1].Attrs().Priority).To(Equal(uint16(16)))
Expect(filters[1].Attrs().Parent).To(Equal(qdiscs[0].Attrs().Handle))
Expect(filters[1].(*netlink.U32).ClassId).To(Equal(netlink.MakeHandle(1, 1)))
filterSel = filters[1].(*netlink.U32).Sel
Expect(filterSel).To(BeAssignableToTypeOf(&netlink.TcU32Sel{}))
Expect(filterSel.Flags).To(Equal(uint8(netlink.TC_U32_TERMINAL)))
Expect(filterSel.Keys).To(HaveLen(1))
Expect(filterSel.Nkeys).To(Equal(uint8(1)))
// The filter should match to 10.0.0.0/8 dst address in other words it should be:
// match 0a000000/ff000000 at 16
selKey := filterSel.Keys[0]
Expect(selKey.Val).To(Equal(uint32(10 * math.Pow(256, 3))))
Expect(selKey.Off).To(Equal(int32(16)))
Expect(selKey.Mask).To(Equal(uint32(255 * math.Pow(256, 3))))
return nil
})).To(Succeed())
})
})
It(fmt.Sprintf("[%s] works with a Veth pair with some ipv4 and ipv6 shaped traffic for specific subnets", ver), func() {
conf := fmt.Sprintf(`{
"cniVersion": "%s",
"name": "cni-plugin-bandwidth-test",
"type": "bandwidth",
"ingressRate": 8,
"ingressBurst": 8,
"egressRate": 16,
"egressBurst": 12,
"shapedSubnets": [
"10.0.0.0/8",
"fd00:db8:abcd:1234:e000::/68"
],
"prevResult": {
"interfaces": [
{
"name": "%s",
"sandbox": ""
},
{
"name": "%s",
"sandbox": "%s"
}
],
"ips": [
{
"version": "4",
"address": "%s/24",
"gateway": "10.0.0.1",
"interface": 1
}
],
"routes": []
}
}`, ver, hostIfname, containerIfname, containerNs.Path(), containerIP.String())
args := &skel.CmdArgs{
ContainerID: "dummy",
Netns: containerNs.Path(),
IfName: containerIfname,
StdinData: []byte(conf),
}
// Container egress (host ingress)
Expect(hostNs.Do(func(netNS ns.NetNS) error {
defer GinkgoRecover()
r, out, err := testutils.CmdAdd(containerNs.Path(), args.ContainerID, "", []byte(conf), func() error { return cmdAdd(args) })
Expect(err).NotTo(HaveOccurred(), string(out))
result, err := types100.GetResult(r)
Expect(err).NotTo(HaveOccurred())
Expect(result.Interfaces).To(HaveLen(3))
Expect(result.Interfaces[2].Name).To(Equal(ifbDeviceName))
Expect(result.Interfaces[2].Sandbox).To(Equal(""))
ifbLink, err := netlink.LinkByName(ifbDeviceName)
Expect(err).NotTo(HaveOccurred())
Expect(ifbLink.Attrs().MTU).To(Equal(hostIfaceMTU))
qdiscs, err := netlink.QdiscList(ifbLink)
Expect(err).NotTo(HaveOccurred())
Expect(qdiscs).To(HaveLen(1))
Expect(qdiscs[0].Attrs().LinkIndex).To(Equal(ifbLink.Attrs().Index))
Expect(qdiscs[0]).To(BeAssignableToTypeOf(&netlink.Htb{}))
Expect(qdiscs[0].(*netlink.Htb).Defcls).To(Equal(uint32(UnShapedClassMinorID)))
classes, err := netlink.ClassList(ifbLink, qdiscs[0].Attrs().Handle)
Expect(err).NotTo(HaveOccurred())
Expect(classes).To(HaveLen(2))
// Uncapped class
Expect(classes[0]).To(BeAssignableToTypeOf(&netlink.HtbClass{}))
Expect(classes[0].(*netlink.HtbClass).Handle).To(Equal(netlink.MakeHandle(1, UnShapedClassMinorID)))
Expect(classes[0].(*netlink.HtbClass).Rate).To(Equal(UncappedRate))
Expect(classes[0].(*netlink.HtbClass).Buffer).To(Equal(uint32(0)))
Expect(classes[0].(*netlink.HtbClass).Ceil).To(Equal(UncappedRate))
Expect(classes[0].(*netlink.HtbClass).Cbuffer).To(Equal(uint32(0)))
// Class with traffic shapping settings
Expect(classes[1]).To(BeAssignableToTypeOf(&netlink.HtbClass{}))
Expect(classes[1].(*netlink.HtbClass).Handle).To(Equal(netlink.MakeHandle(1, ShapedClassMinorID)))
Expect(classes[1].(*netlink.HtbClass).Rate).To(Equal(uint64(2)))
// Expect(classes[1].(*netlink.HtbClass).Buffer).To(Equal(uint32(7812500)))
Expect(classes[1].(*netlink.HtbClass).Ceil).To(Equal(uint64(4)))
// Expect(classes[1].(*netlink.HtbClass).Cbuffer).To(Equal(uint32(0)))
filters, err := netlink.FilterList(ifbLink, qdiscs[0].Attrs().Handle)
Expect(err).NotTo(HaveOccurred())
Expect(filters).To(HaveLen(2))
// traffic to fd00:db8:abcd:1234:e000::/68 redirected to uncapped class
Expect(filters[0]).To(BeAssignableToTypeOf(&netlink.U32{}))
Expect(filters[0].(*netlink.U32).Actions).To(BeEmpty())
Expect(filters[0].Attrs().Protocol).To(Equal(uint16(syscall.ETH_P_IPV6)))
Expect(filters[0].Attrs().LinkIndex).To(Equal(ifbLink.Attrs().Index))
Expect(filters[0].Attrs().Priority).To(Equal(uint16(15)))
Expect(filters[0].Attrs().Parent).To(Equal(qdiscs[0].Attrs().Handle))
Expect(filters[0].(*netlink.U32).ClassId).To(Equal(netlink.MakeHandle(1, ShapedClassMinorID)))
filterSel := filters[0].(*netlink.U32).Sel
Expect(filterSel).To(BeAssignableToTypeOf(&netlink.TcU32Sel{}))
Expect(filterSel.Flags).To(Equal(uint8(netlink.TC_U32_TERMINAL)))
Expect(filterSel.Keys).To(HaveLen(3))
Expect(filterSel.Nkeys).To(Equal(uint8(3)))
// The filter should match to fd00:db8:abcd:1234:e000::/68 dst address in other words it should be:
// match 0xfd000db8/0xffffffff at 24
// match 0xabcd1234/0xffffffff at 28
// match 0xe0000000/0xf0000000 at 32
// (and last match discarded because it would be equivalent to a matchall/true condition at 36)
Expect(filterSel.Keys[0].Off).To(Equal(int32(24)))
Expect(filterSel.Keys[0].Val).To(Equal(uint32(4244639160)))
Expect(filterSel.Keys[0].Mask).To(Equal(uint32(4294967295)))
Expect(filterSel.Keys[1].Off).To(Equal(int32(28)))
Expect(filterSel.Keys[1].Val).To(Equal(uint32(2882343476)))
Expect(filterSel.Keys[1].Mask).To(Equal(uint32(4294967295)))
Expect(filterSel.Keys[2].Off).To(Equal(int32(32)))
Expect(filterSel.Keys[2].Val).To(Equal(uint32(3758096384)))
Expect(filterSel.Keys[2].Mask).To(Equal(uint32(4026531840)))
// traffic to 10.0.0.0/8 redirected to uncapped class
Expect(filters[1]).To(BeAssignableToTypeOf(&netlink.U32{}))
Expect(filters[1].(*netlink.U32).Actions).To(BeEmpty())
Expect(filters[1].Attrs().Protocol).To(Equal(uint16(syscall.ETH_P_IP)))
Expect(filters[1].Attrs().LinkIndex).To(Equal(ifbLink.Attrs().Index))
Expect(filters[1].Attrs().Priority).To(Equal(uint16(16)))
Expect(filters[1].Attrs().Parent).To(Equal(qdiscs[0].Attrs().Handle))
Expect(filters[1].(*netlink.U32).ClassId).To(Equal(netlink.MakeHandle(1, ShapedClassMinorID)))
filterSel = filters[1].(*netlink.U32).Sel
Expect(filterSel).To(BeAssignableToTypeOf(&netlink.TcU32Sel{}))
Expect(filterSel.Flags).To(Equal(uint8(netlink.TC_U32_TERMINAL)))
Expect(filterSel.Keys).To(HaveLen(1))
Expect(filterSel.Nkeys).To(Equal(uint8(1)))
// The filter should match to 10.0.0.0/8 dst address in other words it should be:
// match 0a000000/ff000000 at 16
selKey := filterSel.Keys[0]
Expect(selKey.Val).To(Equal(uint32(10 * math.Pow(256, 3))))
Expect(selKey.Off).To(Equal(int32(16)))
Expect(selKey.Mask).To(Equal(uint32(255 * math.Pow(256, 3))))
hostVethLink, err := netlink.LinkByName(hostIfname)
Expect(err).NotTo(HaveOccurred())
qdiscFilters, err := netlink.FilterList(hostVethLink, netlink.MakeHandle(0xffff, 0))
Expect(err).NotTo(HaveOccurred())
Expect(qdiscFilters).To(HaveLen(1))
Expect(qdiscFilters[0].(*netlink.U32).Actions[0].(*netlink.MirredAction).Ifindex).To(Equal(ifbLink.Attrs().Index))
return nil
})).To(Succeed())
// Container ingress (host egress)
Expect(hostNs.Do(func(n ns.NetNS) error {
defer GinkgoRecover()
vethLink, err := netlink.LinkByName(hostIfname)
Expect(err).NotTo(HaveOccurred())
qdiscs, err := netlink.QdiscList(vethLink)
Expect(err).NotTo(HaveOccurred())
Expect(qdiscs).To(HaveLen(2))
Expect(qdiscs[0].Attrs().LinkIndex).To(Equal(vethLink.Attrs().Index))
Expect(qdiscs[0]).To(BeAssignableToTypeOf(&netlink.Htb{}))
Expect(qdiscs[0].(*netlink.Htb).Defcls).To(Equal(uint32(UnShapedClassMinorID)))
classes, err := netlink.ClassList(vethLink, qdiscs[0].Attrs().Handle)
Expect(err).NotTo(HaveOccurred())
Expect(classes).To(HaveLen(2))
// Uncapped class
Expect(classes[0]).To(BeAssignableToTypeOf(&netlink.HtbClass{}))
Expect(classes[0].(*netlink.HtbClass).Handle).To(Equal(netlink.MakeHandle(1, UnShapedClassMinorID)))
Expect(classes[0].(*netlink.HtbClass).Rate).To(Equal(UncappedRate))
Expect(classes[0].(*netlink.HtbClass).Buffer).To(Equal(uint32(0)))
Expect(classes[0].(*netlink.HtbClass).Ceil).To(Equal(UncappedRate))
Expect(classes[0].(*netlink.HtbClass).Cbuffer).To(Equal(uint32(0)))
// Class with traffic shapping settings
Expect(classes[1]).To(BeAssignableToTypeOf(&netlink.HtbClass{}))
Expect(classes[1].(*netlink.HtbClass).Handle).To(Equal(netlink.MakeHandle(1, ShapedClassMinorID)))
Expect(classes[1].(*netlink.HtbClass).Rate).To(Equal(uint64(1)))
// Expect(classes[1].(*netlink.HtbClass).Buffer).To(Equal(uint32(15625000)))
Expect(classes[1].(*netlink.HtbClass).Ceil).To(Equal(uint64(2)))
// Expect(classes[1].(*netlink.HtbClass).Cbuffer).To(Equal(uint32(0)))
filters, err := netlink.FilterList(vethLink, qdiscs[0].Attrs().Handle)
Expect(err).NotTo(HaveOccurred())
Expect(filters).To(HaveLen(2))
// traffic to fd00:db8:abcd:1234:e000::/68 redirected to uncapped class
Expect(filters[0]).To(BeAssignableToTypeOf(&netlink.U32{}))
Expect(filters[0].(*netlink.U32).Actions).To(BeEmpty())
Expect(filters[0].Attrs().Protocol).To(Equal(uint16(syscall.ETH_P_IPV6)))
Expect(filters[0].Attrs().LinkIndex).To(Equal(vethLink.Attrs().Index))
Expect(filters[0].Attrs().Priority).To(Equal(uint16(15)))
Expect(filters[0].Attrs().Parent).To(Equal(qdiscs[0].Attrs().Handle))
Expect(filters[0].(*netlink.U32).ClassId).To(Equal(netlink.MakeHandle(1, ShapedClassMinorID)))
filterSel := filters[0].(*netlink.U32).Sel
Expect(filterSel).To(BeAssignableToTypeOf(&netlink.TcU32Sel{}))
Expect(filterSel.Flags).To(Equal(uint8(netlink.TC_U32_TERMINAL)))
Expect(filterSel.Keys).To(HaveLen(3))
Expect(filterSel.Nkeys).To(Equal(uint8(3)))
// The filter should match to fd00:db8:abcd:1234:e000::/68 dst address in other words it should be:
// match 0xfd000db8/0xffffffff at 24
// match 0xabcd1234/0xffffffff at 28
// match 0xe0000000/0xf0000000 at 32
// (and last match discarded because it would be equivalent to a matchall/true condition at 36)
Expect(filterSel.Keys[0].Off).To(Equal(int32(24)))
Expect(filterSel.Keys[0].Val).To(Equal(uint32(4244639160)))
Expect(filterSel.Keys[0].Mask).To(Equal(uint32(4294967295)))
Expect(filterSel.Keys[1].Off).To(Equal(int32(28)))
Expect(filterSel.Keys[1].Val).To(Equal(uint32(2882343476)))
Expect(filterSel.Keys[1].Mask).To(Equal(uint32(4294967295)))
Expect(filterSel.Keys[2].Off).To(Equal(int32(32)))
Expect(filterSel.Keys[2].Val).To(Equal(uint32(3758096384)))
Expect(filterSel.Keys[2].Mask).To(Equal(uint32(4026531840)))
// traffic to 10.0.0.0/8 redirected to uncapped class
Expect(filters[1]).To(BeAssignableToTypeOf(&netlink.U32{}))
Expect(filters[1].(*netlink.U32).Actions).To(BeEmpty())
Expect(filters[1].Attrs().Protocol).To(Equal(uint16(syscall.ETH_P_IP)))
Expect(filters[1].Attrs().LinkIndex).To(Equal(vethLink.Attrs().Index))
Expect(filters[1].Attrs().Priority).To(Equal(uint16(16)))
Expect(filters[1].Attrs().Parent).To(Equal(qdiscs[0].Attrs().Handle))
Expect(filters[1].(*netlink.U32).ClassId).To(Equal(netlink.MakeHandle(1, ShapedClassMinorID)))
filterSel = filters[1].(*netlink.U32).Sel
Expect(filterSel).To(BeAssignableToTypeOf(&netlink.TcU32Sel{}))
Expect(filterSel.Flags).To(Equal(uint8(netlink.TC_U32_TERMINAL)))
Expect(filterSel.Keys).To(HaveLen(1))
Expect(filterSel.Nkeys).To(Equal(uint8(1)))
// The filter should match to 10.0.0.0/8 dst address in other words it should be:
// match 0a000000/ff000000 at 16
selKey := filterSel.Keys[0]
Expect(selKey.Val).To(Equal(uint32(10 * math.Pow(256, 3))))
Expect(selKey.Off).To(Equal(int32(16)))
Expect(selKey.Mask).To(Equal(uint32(255 * math.Pow(256, 3))))
return nil
})).To(Succeed())
})
It(fmt.Sprintf("[%s] does not apply ingress when disabled", ver), func() {
conf := fmt.Sprintf(`{
"cniVersion": "%s",
"name": "cni-plugin-bandwidth-test",
"type": "bandwidth",
"ingressRate": 0,
"ingressBurst": 0,
"egressRate": 8000,
"egressBurst": 80,
"unshapedSubnets": [
"10.0.0.0/8",
"fd00:db8:abcd:1234:e000::/68"
],
"prevResult": {
"interfaces": [
{
"name": "%s",
"sandbox": ""
},
{
"name": "%s",
"sandbox": "%s"
}
],
"ips": [
{
"version": "4",
"address": "%s/24",
"gateway": "10.0.0.1",
"interface": 1
}
],
"routes": []
}
}`, ver, hostIfname, containerIfname, containerNs.Path(), containerIP.String())
args := &skel.CmdArgs{
ContainerID: "dummy",
Netns: containerNs.Path(),
IfName: containerIfname,
StdinData: []byte(conf),
}
// check container egress side / host ingress side, we expect to get some QoS setup there
Expect(hostNs.Do(func(netNS ns.NetNS) error {
defer GinkgoRecover()
_, out, err := testutils.CmdAdd(containerNs.Path(), args.ContainerID, ifbDeviceName, []byte(conf), func() error { return cmdAdd(args) })
Expect(err).NotTo(HaveOccurred(), string(out))
ifbLink, err := netlink.LinkByName(ifbDeviceName)
Expect(err).NotTo(HaveOccurred())
qdiscs, err := netlink.QdiscList(ifbLink)
Expect(err).NotTo(HaveOccurred())
Expect(qdiscs).To(HaveLen(1))
Expect(qdiscs[0].Attrs().LinkIndex).To(Equal(ifbLink.Attrs().Index))
Expect(qdiscs[0]).To(BeAssignableToTypeOf(&netlink.Htb{}))
Expect(qdiscs[0].(*netlink.Htb).Defcls).To(Equal(uint32(ShapedClassMinorID)))
classes, err := netlink.ClassList(ifbLink, qdiscs[0].Attrs().Handle)
Expect(err).NotTo(HaveOccurred())
Expect(classes).To(HaveLen(2))
// Uncapped class
Expect(classes[0]).To(BeAssignableToTypeOf(&netlink.HtbClass{}))
Expect(classes[0].(*netlink.HtbClass).Handle).To(Equal(netlink.MakeHandle(1, 1)))
Expect(classes[0].(*netlink.HtbClass).Rate).To(Equal(UncappedRate))
Expect(classes[0].(*netlink.HtbClass).Buffer).To(Equal(uint32(0)))
Expect(classes[0].(*netlink.HtbClass).Ceil).To(Equal(UncappedRate))
Expect(classes[0].(*netlink.HtbClass).Cbuffer).To(Equal(uint32(0)))
// Class with traffic shapping settings
Expect(classes[1]).To(BeAssignableToTypeOf(&netlink.HtbClass{}))
Expect(classes[1].(*netlink.HtbClass).Handle).To(Equal(netlink.MakeHandle(1, ShapedClassMinorID)))
Expect(classes[1].(*netlink.HtbClass).Rate).To(Equal(uint64(1000)))
// Expect(classes[1].(*netlink.HtbClass).Buffer).To(Equal(uint32(7812500)))
Expect(classes[1].(*netlink.HtbClass).Ceil).To(Equal(uint64(2000)))
// Expect(classes[1].(*netlink.HtbClass).Cbuffer).To(Equal(uint32(0)))
filters, err := netlink.FilterList(ifbLink, qdiscs[0].Attrs().Handle)
Expect(err).NotTo(HaveOccurred())
Expect(filters).To(HaveLen(2))
// traffic to fd00:db8:abcd:1234:e000::/68 redirected to uncapped class
Expect(filters[0]).To(BeAssignableToTypeOf(&netlink.U32{}))
Expect(filters[0].(*netlink.U32).Actions).To(BeEmpty())
Expect(filters[0].Attrs().Protocol).To(Equal(uint16(syscall.ETH_P_IPV6)))
Expect(filters[0].Attrs().LinkIndex).To(Equal(ifbLink.Attrs().Index))
Expect(filters[0].Attrs().Priority).To(Equal(uint16(15)))
Expect(filters[0].Attrs().Parent).To(Equal(qdiscs[0].Attrs().Handle))
Expect(filters[0].(*netlink.U32).ClassId).To(Equal(netlink.MakeHandle(1, 1)))
filterSel := filters[0].(*netlink.U32).Sel
Expect(filterSel).To(BeAssignableToTypeOf(&netlink.TcU32Sel{}))
Expect(filterSel.Flags).To(Equal(uint8(netlink.TC_U32_TERMINAL)))
Expect(filterSel.Keys).To(HaveLen(3))
Expect(filterSel.Nkeys).To(Equal(uint8(3)))
// The filter should match to fd00:db8:abcd:1234:e000::/68 dst address in other words it should be:
// match 0xfd000db8/0xffffffff at 24
// match 0xabcd1234/0xffffffff at 28
// match 0xe0000000/0xf0000000 at 32
// (and last match discarded because it would be equivalent to a matchall/true condition at 36)
Expect(filterSel.Keys[0].Off).To(Equal(int32(24)))
Expect(filterSel.Keys[0].Val).To(Equal(uint32(4244639160)))
Expect(filterSel.Keys[0].Mask).To(Equal(uint32(4294967295)))
Expect(filterSel.Keys[1].Off).To(Equal(int32(28)))
Expect(filterSel.Keys[1].Val).To(Equal(uint32(2882343476)))
Expect(filterSel.Keys[1].Mask).To(Equal(uint32(4294967295)))
Expect(filterSel.Keys[2].Off).To(Equal(int32(32)))
Expect(filterSel.Keys[2].Val).To(Equal(uint32(3758096384)))
Expect(filterSel.Keys[2].Mask).To(Equal(uint32(4026531840)))
// traffic to 10.0.0.0/8 redirected to uncapped class
Expect(filters[1]).To(BeAssignableToTypeOf(&netlink.U32{}))
Expect(filters[1].(*netlink.U32).Actions).To(BeEmpty())
Expect(filters[1].Attrs().Protocol).To(Equal(uint16(syscall.ETH_P_IP)))
Expect(filters[1].Attrs().LinkIndex).To(Equal(ifbLink.Attrs().Index))
Expect(filters[1].Attrs().Priority).To(Equal(uint16(16)))
Expect(filters[1].Attrs().Parent).To(Equal(qdiscs[0].Attrs().Handle))
Expect(filters[1].(*netlink.U32).ClassId).To(Equal(netlink.MakeHandle(1, 1)))
filterSel = filters[1].(*netlink.U32).Sel
Expect(filterSel).To(BeAssignableToTypeOf(&netlink.TcU32Sel{}))
Expect(filterSel.Flags).To(Equal(uint8(netlink.TC_U32_TERMINAL)))
Expect(filterSel.Keys).To(HaveLen(1))
Expect(filterSel.Nkeys).To(Equal(uint8(1)))
// The filter should match to 10.0.0.0/8 dst address in other words it should be:
// match 0a000000/ff000000 at 16
selKey := filterSel.Keys[0]
Expect(selKey.Val).To(Equal(uint32(10 * math.Pow(256, 3))))
Expect(selKey.Off).To(Equal(int32(16)))
Expect(selKey.Mask).To(Equal(uint32(255 * math.Pow(256, 3))))
// check traffic mirroring from veth to ifb
hostVethLink, err := netlink.LinkByName(hostIfname)
Expect(err).NotTo(HaveOccurred())
qdiscFilters, err := netlink.FilterList(hostVethLink, netlink.MakeHandle(0xffff, 0))
Expect(err).NotTo(HaveOccurred())
Expect(qdiscFilters).To(HaveLen(1))
Expect(qdiscFilters[0].(*netlink.U32).Actions[0].(*netlink.MirredAction).Ifindex).To(Equal(ifbLink.Attrs().Index))
return nil
})).To(Succeed())
// check container ingress side / host egress side, we should not have any htb qdisc/classes/filters defined for the host veth
// only the qdisc ingress + a noqueue qdisc
Expect(hostNs.Do(func(n ns.NetNS) error {
defer GinkgoRecover()
containerIfLink, err := netlink.LinkByName(hostIfname)
Expect(err).NotTo(HaveOccurred())
qdiscs, err := netlink.QdiscList(containerIfLink)
Expect(err).NotTo(HaveOccurred())
Expect(qdiscs).To(HaveLen(2))
Expect(qdiscs[0]).NotTo(BeAssignableToTypeOf(&netlink.Htb{}))
Expect(qdiscs[1]).NotTo(BeAssignableToTypeOf(&netlink.Htb{}))
return nil
})).To(Succeed())
})
It(fmt.Sprintf("[%s] does not apply egress when disabled", ver), func() {
conf := fmt.Sprintf(`{
"cniVersion": "%s",
"name": "cni-plugin-bandwidth-test",
"type": "bandwidth",
"egressRate": 0,
"egressBurst": 0,
"ingressRate": 8000,
"ingressBurst": 80,
"prevResult": {
"interfaces": [
{
"name": "%s",
"sandbox": ""
},
{
"name": "%s",
"sandbox": "%s"
}
],
"ips": [
{
"version": "4",
"address": "%s/24",
"gateway": "10.0.0.1",
"interface": 1
}
],
"routes": []
}
}`, ver, hostIfname, containerIfname, containerNs.Path(), containerIP.String())
args := &skel.CmdArgs{
ContainerID: "dummy",
Netns: containerNs.Path(),
IfName: containerIfname,
StdinData: []byte(conf),
}
Expect(hostNs.Do(func(netNS ns.NetNS) error {
defer GinkgoRecover()
_, out, err := testutils.CmdAdd(containerNs.Path(), args.ContainerID, ifbDeviceName, []byte(conf), func() error { return cmdAdd(args) })
Expect(err).NotTo(HaveOccurred(), string(out))
// Since we do not setup any egress QoS, no ifb interface should be created at all
_, err = netlink.LinkByName(ifbDeviceName)
Expect(err).To(HaveOccurred())
return nil
})).To(Succeed())
Expect(hostNs.Do(func(n ns.NetNS) error {
defer GinkgoRecover()
containerIfLink, err := netlink.LinkByName(hostIfname)
Expect(err).NotTo(HaveOccurred())
// Only one qdisc should be found this time, no ingress qdisc should be there
qdiscs, err := netlink.QdiscList(containerIfLink)
Expect(err).NotTo(HaveOccurred())
Expect(qdiscs).To(HaveLen(1))
Expect(qdiscs[0].Attrs().LinkIndex).To(Equal(containerIfLink.Attrs().Index))
Expect(qdiscs[0]).To(BeAssignableToTypeOf(&netlink.Htb{}))
Expect(qdiscs[0].(*netlink.Htb).Defcls).To(Equal(uint32(ShapedClassMinorID)))
classes, err := netlink.ClassList(containerIfLink, qdiscs[0].Attrs().Handle)
Expect(err).NotTo(HaveOccurred())
Expect(classes).To(HaveLen(2))
// Uncapped class
Expect(classes[0]).To(BeAssignableToTypeOf(&netlink.HtbClass{}))
Expect(classes[0].(*netlink.HtbClass).Handle).To(Equal(netlink.MakeHandle(1, 1)))
Expect(classes[0].(*netlink.HtbClass).Rate).To(Equal(UncappedRate))
Expect(classes[0].(*netlink.HtbClass).Buffer).To(Equal(uint32(0)))
Expect(classes[0].(*netlink.HtbClass).Ceil).To(Equal(UncappedRate))
Expect(classes[0].(*netlink.HtbClass).Cbuffer).To(Equal(uint32(0)))
// Class with traffic shapping settings
Expect(classes[1]).To(BeAssignableToTypeOf(&netlink.HtbClass{}))
Expect(classes[1].(*netlink.HtbClass).Handle).To(Equal(netlink.MakeHandle(1, uint16(qdiscs[0].(*netlink.Htb).Defcls))))
Expect(classes[1].(*netlink.HtbClass).Rate).To(Equal(uint64(1000)))
// Expect(classes[1].(*netlink.HtbClass).Buffer).To(Equal(uint32(15625000)))
Expect(classes[1].(*netlink.HtbClass).Ceil).To(Equal(uint64(2000)))
// Expect(classes[1].(*netlink.HtbClass).Cbuffer).To(Equal(uint32(0)))
// No subnets are exluded in this example so we should not get any filter
filters, err := netlink.FilterList(containerIfLink, qdiscs[0].Attrs().Handle)
Expect(err).NotTo(HaveOccurred())
Expect(filters).To(BeEmpty())
// Just check no mirroring is setup
qdiscFilters, err := netlink.FilterList(containerIfLink, netlink.MakeHandle(0xffff, 0))
Expect(err).NotTo(HaveOccurred())
Expect(qdiscFilters).To(BeEmpty())
return nil
})).To(Succeed())
})
It(fmt.Sprintf("[%s] fails with invalid UnshapedSubnets", ver), func() {
conf := fmt.Sprintf(`{
"cniVersion": "%s",
"name": "cni-plugin-bandwidth-test",
"type": "bandwidth",
"ingressRate": 123,
"ingressBurst": 123,
"egressRate": 123,
"egressBurst": 123,
"unshapedSubnets": ["10.0.0.0/8", "hello"],
"prevResult": {
"interfaces": [
{
"name": "%s",
"sandbox": ""
},
{
"name": "%s",
"sandbox": "%s"
}
],
"ips": [
{
"version": "4",
"address": "%s/24",
"gateway": "10.0.0.1",
"interface": 1
}
],
"routes": []
}
}`, ver, hostIfname, containerIfname, containerNs.Path(), containerIP.String())
args := &skel.CmdArgs{
ContainerID: "dummy",
Netns: containerNs.Path(),
IfName: "eth0",
StdinData: []byte(conf),
}
Expect(hostNs.Do(func(netNS ns.NetNS) error {
defer GinkgoRecover()
_, _, err := testutils.CmdAdd(containerNs.Path(), args.ContainerID, "", []byte(conf), func() error { return cmdAdd(args) })
Expect(err).To(MatchError("bad subnet \"hello\" provided, details invalid CIDR address: hello"))
return nil
})).To(Succeed())
})
It(fmt.Sprintf("[%s] fails with invalid ShapedSubnets", ver), func() {
conf := fmt.Sprintf(`{
"cniVersion": "%s",
"name": "cni-plugin-bandwidth-test",
"type": "bandwidth",
"ingressRate": 123,
"ingressBurst": 123,
"egressRate": 123,
"egressBurst": 123,
"ShapedSubnets": ["10.0.0.0/8", "hello"],
"prevResult": {
"interfaces": [
{
"name": "%s",
"sandbox": ""
},
{
"name": "%s",
"sandbox": "%s"
}
],
"ips": [
{
"version": "4",
"address": "%s/24",
"gateway": "10.0.0.1",
"interface": 1
}
],
"routes": []
}
}`, ver, hostIfname, containerIfname, containerNs.Path(), containerIP.String())
args := &skel.CmdArgs{
ContainerID: "dummy",
Netns: containerNs.Path(),
IfName: "eth0",
StdinData: []byte(conf),
}
Expect(hostNs.Do(func(netNS ns.NetNS) error {
defer GinkgoRecover()
_, _, err := testutils.CmdAdd(containerNs.Path(), args.ContainerID, "", []byte(conf), func() error { return cmdAdd(args) })
Expect(err).To(MatchError("bad subnet \"hello\" provided, details invalid CIDR address: hello"))
return nil
})).To(Succeed())
})
It(fmt.Sprintf("[%s] fails with both ShapedSubnets and UnShapedSubnets specified", ver), func() {
conf := fmt.Sprintf(`{
"cniVersion": "%s",
"name": "cni-plugin-bandwidth-test",
"type": "bandwidth",
"ingressRate": 123,
"ingressBurst": 123,
"egressRate": 123,
"egressBurst": 123,
"ShapedSubnets": ["10.0.0.0/8"],
"UnShapedSubnets": ["192.168.0.0/16"],
"prevResult": {
"interfaces": [
{
"name": "%s",
"sandbox": ""
},
{
"name": "%s",
"sandbox": "%s"
}
],
"ips": [
{
"version": "4",
"address": "%s/24",
"gateway": "10.0.0.1",
"interface": 1
}
],
"routes": []
}
}`, ver, hostIfname, containerIfname, containerNs.Path(), containerIP.String())
args := &skel.CmdArgs{
ContainerID: "dummy",
Netns: containerNs.Path(),
IfName: "eth0",
StdinData: []byte(conf),
}
Expect(hostNs.Do(func(netNS ns.NetNS) error {
defer GinkgoRecover()
_, _, err := testutils.CmdAdd(containerNs.Path(), args.ContainerID, "", []byte(conf), func() error { return cmdAdd(args) })
Expect(err).To(MatchError("unshapedSubnets and shapedSubnets cannot be both specified, one of them should be discarded"))
return nil
})).To(Succeed())
})
It(fmt.Sprintf("[%s] fails an invalid ingress config", ver), func() {
conf := fmt.Sprintf(`{
"cniVersion": "%s",
"name": "cni-plugin-bandwidth-test",
"type": "bandwidth",
"ingressRate": 0,
"ingressBurst": 123,
"egressRate": 123,
"egressBurst": 123,
"prevResult": {
"interfaces": [
{
"name": "%s",
"sandbox": ""
},
{
"name": "%s",
"sandbox": "%s"
}
],
"ips": [
{
"version": "4",
"address": "%s/24",
"gateway": "10.0.0.1",
"interface": 1
}
],
"routes": []
}
}`, ver, hostIfname, containerIfname, containerNs.Path(), containerIP.String())
args := &skel.CmdArgs{
ContainerID: "dummy",
Netns: containerNs.Path(),
IfName: "eth0",
StdinData: []byte(conf),
}
Expect(hostNs.Do(func(netNS ns.NetNS) error {
defer GinkgoRecover()
_, _, err := testutils.CmdAdd(containerNs.Path(), args.ContainerID, "", []byte(conf), func() error { return cmdAdd(args) })
Expect(err).To(MatchError("if burst is set, rate must also be set"))
return nil
})).To(Succeed())
})
It(fmt.Sprintf("[%s] fails an invalid egress config", ver), func() {
conf := fmt.Sprintf(`{
"cniVersion": "%s",
"name": "cni-plugin-bandwidth-test",
"type": "bandwidth",
"ingressRate": 123,
"ingressBurst": 123,
"egressRate": 0,
"egressBurst": 123,
"prevResult": {
"interfaces": [
{
"name": "%s",
"sandbox": ""
},
{
"name": "%s",
"sandbox": "%s"
}
],
"ips": [
{
"version": "4",
"address": "%s/24",
"gateway": "10.0.0.1",
"interface": 1
}
],
"routes": []
}
}`, ver, hostIfname, containerIfname, containerNs.Path(), containerIP.String())
args := &skel.CmdArgs{
ContainerID: "dummy",
Netns: containerNs.Path(),
IfName: "eth0",
StdinData: []byte(conf),
}
Expect(hostNs.Do(func(netNS ns.NetNS) error {
defer GinkgoRecover()
_, _, err := testutils.CmdAdd(containerNs.Path(), args.ContainerID, "", []byte(conf), func() error { return cmdAdd(args) })
Expect(err).To(MatchError("if burst is set, rate must also be set"))
return nil
})).To(Succeed())
})
It(fmt.Sprintf("[%s] works with a Veth pair using runtime config", ver), func() {
conf := fmt.Sprintf(`{
"cniVersion": "%s",
"name": "cni-plugin-bandwidth-test",
"type": "bandwidth",
"runtimeConfig": {
"bandWidth": {
"ingressRate": 8,
"ingressBurst": 8,
"egressRate": 16,
"egressBurst": 9,
"unshapedSubnets": ["192.168.0.0/24"]
}
},
"prevResult": {
"interfaces": [
{
"name": "%s",
"sandbox": ""
},
{
"name": "%s",
"sandbox": "%s"
}
],
"ips": [
{
"version": "4",
"address": "%s/24",
"gateway": "10.0.0.1",
"interface": 1
}
],
"routes": []
}
}`, ver, hostIfname, containerIfname, containerNs.Path(), containerIP.String())
args := &skel.CmdArgs{
ContainerID: "dummy",
Netns: containerNs.Path(),
IfName: containerIfname,
StdinData: []byte(conf),
}
Expect(hostNs.Do(func(netNS ns.NetNS) error {
defer GinkgoRecover()
r, out, err := testutils.CmdAdd(containerNs.Path(), args.ContainerID, "", []byte(conf), func() error { return cmdAdd(args) })
Expect(err).NotTo(HaveOccurred(), string(out))
result, err := types100.GetResult(r)
Expect(err).NotTo(HaveOccurred())
Expect(result.Interfaces).To(HaveLen(3))
Expect(result.Interfaces[2].Name).To(Equal(ifbDeviceName))
Expect(result.Interfaces[2].Sandbox).To(Equal(""))
ifbLink, err := netlink.LinkByName(ifbDeviceName)
Expect(err).NotTo(HaveOccurred())
Expect(ifbLink.Attrs().MTU).To(Equal(hostIfaceMTU))
qdiscs, err := netlink.QdiscList(ifbLink)
Expect(err).NotTo(HaveOccurred())
Expect(qdiscs).To(HaveLen(1))
Expect(qdiscs[0].Attrs().LinkIndex).To(Equal(ifbLink.Attrs().Index))
Expect(qdiscs[0]).To(BeAssignableToTypeOf(&netlink.Htb{}))
Expect(qdiscs[0].(*netlink.Htb).Defcls).To(Equal(uint32(ShapedClassMinorID)))
classes, err := netlink.ClassList(ifbLink, qdiscs[0].Attrs().Handle)
Expect(err).NotTo(HaveOccurred())
Expect(classes).To(HaveLen(2))
// Uncapped class
Expect(classes[0]).To(BeAssignableToTypeOf(&netlink.HtbClass{}))
Expect(classes[0].(*netlink.HtbClass).Handle).To(Equal(netlink.MakeHandle(1, 1)))
Expect(classes[0].(*netlink.HtbClass).Rate).To(Equal(UncappedRate))
Expect(classes[0].(*netlink.HtbClass).Buffer).To(Equal(uint32(0)))
Expect(classes[0].(*netlink.HtbClass).Ceil).To(Equal(UncappedRate))
Expect(classes[0].(*netlink.HtbClass).Cbuffer).To(Equal(uint32(0)))
// Class with traffic shapping settings
Expect(classes[1]).To(BeAssignableToTypeOf(&netlink.HtbClass{}))
Expect(classes[1].(*netlink.HtbClass).Handle).To(Equal(netlink.MakeHandle(1, uint16(qdiscs[0].(*netlink.Htb).Defcls))))
Expect(classes[1].(*netlink.HtbClass).Rate).To(Equal(uint64(2)))
// Expect(classes[1].(*netlink.HtbClass).Buffer).To(Equal(uint32(7812500)))
Expect(classes[1].(*netlink.HtbClass).Ceil).To(Equal(uint64(4)))
// Expect(classes[1].(*netlink.HtbClass).Cbuffer).To(Equal(uint32(0)))
filters, err := netlink.FilterList(ifbLink, qdiscs[0].Attrs().Handle)
Expect(err).NotTo(HaveOccurred())
Expect(filters).To(HaveLen(1))
// traffic to 192.168.0.0/24 redirected to uncapped class
Expect(filters[0]).To(BeAssignableToTypeOf(&netlink.U32{}))
Expect(filters[0].(*netlink.U32).Actions).To(BeEmpty())
Expect(filters[0].Attrs().Protocol).To(Equal(uint16(syscall.ETH_P_IP)))
Expect(filters[0].Attrs().LinkIndex).To(Equal(ifbLink.Attrs().Index))
Expect(filters[0].Attrs().Priority).To(Equal(uint16(16)))
Expect(filters[0].Attrs().Parent).To(Equal(qdiscs[0].Attrs().Handle))
Expect(filters[0].(*netlink.U32).ClassId).To(Equal(netlink.MakeHandle(1, 1)))
filterSel := filters[0].(*netlink.U32).Sel
Expect(filterSel).To(BeAssignableToTypeOf(&netlink.TcU32Sel{}))
Expect(filterSel.Flags).To(Equal(uint8(netlink.TC_U32_TERMINAL)))
Expect(filterSel.Keys).To(HaveLen(1))
Expect(filterSel.Nkeys).To(Equal(uint8(1)))
// The filter should match to 192.168.0.0/24 dst address in other words it should be:
// match c0a80000/ffffff00 at 16
selKey := filterSel.Keys[0]
Expect(selKey.Val).To(Equal(uint32(192*math.Pow(256, 3) + 168*math.Pow(256, 2))))
Expect(selKey.Off).To(Equal(int32(16)))
Expect(selKey.Mask).To(Equal(uint32(255*math.Pow(256, 3) + 255*math.Pow(256, 2) + 255*256)))
hostVethLink, err := netlink.LinkByName(hostIfname)
Expect(err).NotTo(HaveOccurred())
qdiscFilters, err := netlink.FilterList(hostVethLink, netlink.MakeHandle(0xffff, 0))
Expect(err).NotTo(HaveOccurred())
Expect(qdiscFilters).To(HaveLen(1))
Expect(qdiscFilters[0].(*netlink.U32).Actions[0].(*netlink.MirredAction).Ifindex).To(Equal(ifbLink.Attrs().Index))
return nil
})).To(Succeed())
// Container ingress (host egress)
Expect(hostNs.Do(func(n ns.NetNS) error {
defer GinkgoRecover()
vethLink, err := netlink.LinkByName(hostIfname)
Expect(err).NotTo(HaveOccurred())
qdiscs, err := netlink.QdiscList(vethLink)
Expect(err).NotTo(HaveOccurred())
Expect(qdiscs).To(HaveLen(2))
Expect(qdiscs[0].Attrs().LinkIndex).To(Equal(vethLink.Attrs().Index))
Expect(qdiscs[0]).To(BeAssignableToTypeOf(&netlink.Htb{}))
Expect(qdiscs[0].(*netlink.Htb).Defcls).To(Equal(uint32(ShapedClassMinorID)))
classes, err := netlink.ClassList(vethLink, qdiscs[0].Attrs().Handle)
Expect(err).NotTo(HaveOccurred())
Expect(classes).To(HaveLen(2))
// Uncapped class
Expect(classes[0]).To(BeAssignableToTypeOf(&netlink.HtbClass{}))
Expect(classes[0].(*netlink.HtbClass).Handle).To(Equal(netlink.MakeHandle(1, 1)))
Expect(classes[0].(*netlink.HtbClass).Rate).To(Equal(UncappedRate))
Expect(classes[0].(*netlink.HtbClass).Buffer).To(Equal(uint32(0)))
Expect(classes[0].(*netlink.HtbClass).Ceil).To(Equal(UncappedRate))
Expect(classes[0].(*netlink.HtbClass).Cbuffer).To(Equal(uint32(0)))
// Class with traffic shapping settings
Expect(classes[1]).To(BeAssignableToTypeOf(&netlink.HtbClass{}))
Expect(classes[1].(*netlink.HtbClass).Handle).To(Equal(netlink.MakeHandle(1, uint16(qdiscs[0].(*netlink.Htb).Defcls))))
Expect(classes[1].(*netlink.HtbClass).Rate).To(Equal(uint64(1)))
// Expect(classes[1].(*netlink.HtbClass).Buffer).To(Equal(uint32(15625000)))
Expect(classes[1].(*netlink.HtbClass).Ceil).To(Equal(uint64(2)))
// Expect(classes[1].(*netlink.HtbClass).Cbuffer).To(Equal(uint32(0)))
filters, err := netlink.FilterList(vethLink, qdiscs[0].Attrs().Handle)
Expect(err).NotTo(HaveOccurred())
Expect(filters).To(HaveLen(1))
// traffic to 192.168.0.0/24 redirected to uncapped class
Expect(filters[0]).To(BeAssignableToTypeOf(&netlink.U32{}))
Expect(filters[0].(*netlink.U32).Actions).To(BeEmpty())
Expect(filters[0].Attrs().Protocol).To(Equal(uint16(syscall.ETH_P_IP)))
Expect(filters[0].Attrs().LinkIndex).To(Equal(vethLink.Attrs().Index))
Expect(filters[0].Attrs().Priority).To(Equal(uint16(16)))
Expect(filters[0].Attrs().Parent).To(Equal(qdiscs[0].Attrs().Handle))
Expect(filters[0].(*netlink.U32).ClassId).To(Equal(netlink.MakeHandle(1, 1)))
filterSel := filters[0].(*netlink.U32).Sel
Expect(filterSel).To(BeAssignableToTypeOf(&netlink.TcU32Sel{}))
Expect(filterSel.Flags).To(Equal(uint8(netlink.TC_U32_TERMINAL)))
Expect(filterSel.Keys).To(HaveLen(1))
Expect(filterSel.Nkeys).To(Equal(uint8(1)))
// The filter should match to 192.168.0.0/24 dst address in other words it should be:
// match c0a80000/ffffff00 at 16
selKey := filterSel.Keys[0]
Expect(selKey.Val).To(Equal(uint32(192*math.Pow(256, 3) + 168*math.Pow(256, 2))))
Expect(selKey.Off).To(Equal(int32(16)))
Expect(selKey.Mask).To(Equal(uint32(255*math.Pow(256, 3) + 255*math.Pow(256, 2) + 255*256)))
return nil
})).To(Succeed())
})
// Runtime config parameters are expected to be preempted by the global config ones whenever specified
It(fmt.Sprintf("[%s] should apply static config when both static config and runtime config exist", ver), func() {
conf := fmt.Sprintf(`{
"cniVersion": "%s",
"name": "cni-plugin-bandwidth-test",
"type": "bandwidth",
"ingressRate": 0,
"ingressBurst": 0,
"egressRate": 123,
"egressBurst": 123,
"unshapedSubnets": ["192.168.0.0/24"],
"runtimeConfig": {
"bandWidth": {
"ingressRate": 8,
"ingressBurst": 8,
"egressRate": 16,
"egressBurst": 9,
"unshapedSubnets": ["10.0.0.0/8", "fd00:db8:abcd:1234:e000::/68"]
}
},
"prevResult": {
"interfaces": [
{
"name": "%s",
"sandbox": ""
},
{
"name": "%s",
"sandbox": "%s"
}
],
"ips": [
{
"version": "4",
"address": "%s/24",
"gateway": "10.0.0.1",
"interface": 1
}
],
"routes": []
}
}`, ver, hostIfname, containerIfname, containerNs.Path(), containerIP.String())
args := &skel.CmdArgs{
ContainerID: "dummy",
Netns: containerNs.Path(),
IfName: containerIfname,
StdinData: []byte(conf),
}
Expect(hostNs.Do(func(netNS ns.NetNS) error {
defer GinkgoRecover()
r, out, err := testutils.CmdAdd(containerNs.Path(), args.ContainerID, "", []byte(conf), func() error { return cmdAdd(args) })
Expect(err).NotTo(HaveOccurred(), string(out))
result, err := types100.GetResult(r)
Expect(err).NotTo(HaveOccurred())
Expect(result.Interfaces).To(HaveLen(3))
Expect(result.Interfaces[2].Name).To(Equal(ifbDeviceName))
Expect(result.Interfaces[2].Sandbox).To(Equal(""))
ifbLink, err := netlink.LinkByName(ifbDeviceName)
Expect(err).NotTo(HaveOccurred())
Expect(ifbLink.Attrs().MTU).To(Equal(hostIfaceMTU))
qdiscs, err := netlink.QdiscList(ifbLink)
Expect(err).NotTo(HaveOccurred())
Expect(qdiscs).To(HaveLen(1))
Expect(qdiscs[0].Attrs().LinkIndex).To(Equal(ifbLink.Attrs().Index))
Expect(qdiscs[0]).To(BeAssignableToTypeOf(&netlink.Htb{}))
Expect(qdiscs[0].(*netlink.Htb).Defcls).To(Equal(uint32(ShapedClassMinorID)))
classes, err := netlink.ClassList(ifbLink, qdiscs[0].Attrs().Handle)
Expect(err).NotTo(HaveOccurred())
Expect(classes).To(HaveLen(2))
// Uncapped class
Expect(classes[0]).To(BeAssignableToTypeOf(&netlink.HtbClass{}))
Expect(classes[0].(*netlink.HtbClass).Handle).To(Equal(netlink.MakeHandle(1, 1)))
Expect(classes[0].(*netlink.HtbClass).Rate).To(Equal(UncappedRate))
Expect(classes[0].(*netlink.HtbClass).Buffer).To(Equal(uint32(0)))
Expect(classes[0].(*netlink.HtbClass).Ceil).To(Equal(UncappedRate))
Expect(classes[0].(*netlink.HtbClass).Cbuffer).To(Equal(uint32(0)))
// Class with traffic shapping settings
Expect(classes[1]).To(BeAssignableToTypeOf(&netlink.HtbClass{}))
Expect(classes[1].(*netlink.HtbClass).Handle).To(Equal(netlink.MakeHandle(1, uint16(qdiscs[0].(*netlink.Htb).Defcls))))
Expect(classes[1].(*netlink.HtbClass).Rate).To(Equal(uint64(15)))
// Expect(classes[1].(*netlink.HtbClass).Buffer).To(Equal(uint32(7812500)))
Expect(classes[1].(*netlink.HtbClass).Ceil).To(Equal(uint64(30)))
// Expect(classes[1].(*netlink.HtbClass).Cbuffer).To(Equal(uint32(0)))
filters, err := netlink.FilterList(ifbLink, qdiscs[0].Attrs().Handle)
Expect(err).NotTo(HaveOccurred())
Expect(filters).To(HaveLen(1))
// traffic to 192.168.0.0/24 redirected to uncapped class
Expect(filters[0]).To(BeAssignableToTypeOf(&netlink.U32{}))
Expect(filters[0].(*netlink.U32).Actions).To(BeEmpty())
Expect(filters[0].Attrs().Protocol).To(Equal(uint16(syscall.ETH_P_IP)))
Expect(filters[0].Attrs().LinkIndex).To(Equal(ifbLink.Attrs().Index))
Expect(filters[0].Attrs().Priority).To(Equal(uint16(16)))
Expect(filters[0].Attrs().Parent).To(Equal(qdiscs[0].Attrs().Handle))
Expect(filters[0].(*netlink.U32).ClassId).To(Equal(netlink.MakeHandle(1, 1)))
filterSel := filters[0].(*netlink.U32).Sel
Expect(filterSel).To(BeAssignableToTypeOf(&netlink.TcU32Sel{}))
Expect(filterSel.Flags).To(Equal(uint8(netlink.TC_U32_TERMINAL)))
Expect(filterSel.Keys).To(HaveLen(1))
Expect(filterSel.Nkeys).To(Equal(uint8(1)))
// The filter should match to 192.168.0.0/24 dst address in other words it should be:
// match c0a80000/ffffff00 at 16
selKey := filterSel.Keys[0]
Expect(selKey.Val).To(Equal(uint32(192*math.Pow(256, 3) + 168*math.Pow(256, 2))))
Expect(selKey.Off).To(Equal(int32(16)))
Expect(selKey.Mask).To(Equal(uint32(255*math.Pow(256, 3) + 255*math.Pow(256, 2) + 255*256)))
hostVethLink, err := netlink.LinkByName(hostIfname)
Expect(err).NotTo(HaveOccurred())
qdiscFilters, err := netlink.FilterList(hostVethLink, netlink.MakeHandle(0xffff, 0))
Expect(err).NotTo(HaveOccurred())
Expect(qdiscFilters).To(HaveLen(1))
Expect(qdiscFilters[0].(*netlink.U32).Actions[0].(*netlink.MirredAction).Ifindex).To(Equal(ifbLink.Attrs().Index))
return nil
})).To(Succeed())
// Container ingress (host egress)
Expect(hostNs.Do(func(n ns.NetNS) error {
defer GinkgoRecover()
vethLink, err := netlink.LinkByName(hostIfname)
Expect(err).NotTo(HaveOccurred())
qdiscs, err := netlink.QdiscList(vethLink)
Expect(err).NotTo(HaveOccurred())
// No ingress QoS just mirroring
Expect(qdiscs).To(HaveLen(2))
Expect(qdiscs[0].Attrs().LinkIndex).To(Equal(vethLink.Attrs().Index))
Expect(qdiscs[0]).NotTo(BeAssignableToTypeOf(&netlink.Htb{}))
Expect(qdiscs[1]).NotTo(BeAssignableToTypeOf(&netlink.Htb{}))
return nil
})).To(Succeed())
})
It(fmt.Sprintf("[%s] should apply static config when both static config and runtime config exist (bad config example)", ver), func() {
conf := fmt.Sprintf(`{
"cniVersion": "%s",
"name": "cni-plugin-bandwidth-test",
"type": "bandwidth",
"ingressRate": 0,
"ingressBurst": 123,
"egressRate": 123,
"egressBurst": 123,
"runtimeConfig": {
"bandWidth": {
"ingressRate": 8,
"ingressBurst": 8,
"egressRate": 16,
"egressBurst": 9
}
},
"prevResult": {
"interfaces": [
{
"name": "%s",
"sandbox": ""
},
{
"name": "%s",
"sandbox": "%s"
}
],
"ips": [
{
"version": "4",
"address": "%s/24",
"gateway": "10.0.0.1",
"interface": 1
}
],
"routes": []
}
}`, ver, hostIfname, containerIfname, containerNs.Path(), containerIP.String())
args := &skel.CmdArgs{
ContainerID: "dummy",
Netns: containerNs.Path(),
IfName: "eth0",
StdinData: []byte(conf),
}
Expect(hostNs.Do(func(netNS ns.NetNS) error {
defer GinkgoRecover()
_, _, err := testutils.CmdAdd(containerNs.Path(), args.ContainerID, "", []byte(conf), func() error { return cmdAdd(args) })
Expect(err).To(MatchError("if burst is set, rate must also be set"))
return nil
})).To(Succeed())
})
Describe("cmdDEL", func() {
It(fmt.Sprintf("[%s] works with a Veth pair using 0.3.0 config", ver), func() {
conf := fmt.Sprintf(`{
"cniVersion": "%s",
"name": "cni-plugin-bandwidth-test",
"type": "bandwidth",
"ingressRate": 8,
"ingressBurst": 8,
"egressRate": 9,
"egressBurst": 9,
"prevResult": {
"interfaces": [
{
"name": "%s",
"sandbox": ""
},
{
"name": "%s",
"sandbox": "%s"
}
],
"ips": [
{
"version": "4",
"address": "%s/24",
"gateway": "10.0.0.1",
"interface": 1
}
],
"routes": []
}
}`, ver, hostIfname, containerIfname, containerNs.Path(), containerIP.String())
args := &skel.CmdArgs{
ContainerID: "dummy",
Netns: containerNs.Path(),
IfName: containerIfname,
StdinData: []byte(conf),
}
Expect(hostNs.Do(func(_ ns.NetNS) error {
defer GinkgoRecover()
_, out, err := testutils.CmdAdd(containerNs.Path(), args.ContainerID, "", []byte(conf), func() error { return cmdAdd(args) })
Expect(err).NotTo(HaveOccurred(), string(out))
_, err = netlink.LinkByName(hostIfname)
Expect(err).NotTo(HaveOccurred())
_, err = netlink.LinkByName(ifbDeviceName)
Expect(err).NotTo(HaveOccurred())
err = testutils.CmdDel(containerNs.Path(), args.ContainerID, "", func() error { return cmdDel(args) })
Expect(err).NotTo(HaveOccurred(), string(out))
_, err = netlink.LinkByName(ifbDeviceName)
Expect(err).To(HaveOccurred())
// The host veth peer should remain as it has not be created by this plugin
_, err = netlink.LinkByName(hostIfname)
Expect(err).NotTo(HaveOccurred())
return nil
})).To(Succeed())
})
})
Describe("cmdCHECK", func() {
It(fmt.Sprintf("[%s] works with a Veth pair", ver), func() {
conf := fmt.Sprintf(`{
"cniVersion": "%s",
"name": "cni-plugin-bandwidth-test",
"type": "bandwidth",
"ingressRate": 8,
"ingressBurst": 8,
"egressRate": 9,
"egressBurst": 9,
"prevResult": {
"interfaces": [
{
"name": "%s",
"sandbox": ""
},
{
"name": "%s",
"sandbox": "%s"
}
],
"ips": [
{
"version": "4",
"address": "%s/24",
"gateway": "10.0.0.1",
"interface": 1
}
],
"routes": []
}
}`, ver, hostIfname, containerIfname, containerNs.Path(), containerIP.String())
args := &skel.CmdArgs{
ContainerID: "dummy",
Netns: containerNs.Path(),
IfName: containerIfname,
StdinData: []byte(conf),
}
Expect(hostNs.Do(func(netNS ns.NetNS) error {
defer GinkgoRecover()
_, out, err := testutils.CmdAdd(containerNs.Path(), args.ContainerID, "", []byte(conf), func() error { return cmdAdd(args) })
Expect(err).NotTo(HaveOccurred(), string(out))
_, err = netlink.LinkByName(hostIfname)
Expect(err).NotTo(HaveOccurred())
_, err = netlink.LinkByName(ifbDeviceName)
Expect(err).NotTo(HaveOccurred())
if testutils.SpecVersionHasCHECK(ver) {
// Do CNI Check
err = testutils.CmdCheck(containerNs.Path(), args.ContainerID, "", func() error { return cmdCheck(args) })
Expect(err).NotTo(HaveOccurred())
}
err = testutils.CmdDel(containerNs.Path(), args.ContainerID, "", func() error { return cmdDel(args) })
Expect(err).NotTo(HaveOccurred(), string(out))
_, err = netlink.LinkByName(ifbDeviceName)
Expect(err).To(HaveOccurred())
// The host veth peer should remain as it has not be created by this plugin
_, err = netlink.LinkByName(hostIfname)
Expect(err).NotTo(HaveOccurred())
return nil
})).To(Succeed())
})
})
Describe("Getting the host interface which plugin should work on from veth peer of container interface", func() {
It(fmt.Sprintf("[%s] should work with multiple host veth interfaces", ver), func() {
// create veth peer in host ns
vethName, peerName := "host-veth-peer1", "host-veth-peer2"
createVethInOneNs(hostNs, vethName, peerName)
conf := fmt.Sprintf(`{
"cniVersion": "%s",
"name": "cni-plugin-bandwidth-test",
"type": "bandwidth",
"ingressRate": 8,
"ingressBurst": 8,
"egressRate": 16,
"egressBurst": 8,
"prevResult": {
"interfaces": [
{
"name": "%s",
"sandbox": ""
},
{
"name": "%s",
"sandbox": ""
},
{
"name": "%s",
"sandbox": ""
},
{
"name": "%s",
"sandbox": "%s"
}
],
"ips": [
{
"version": "4",
"address": "%s/24",
"gateway": "10.0.0.1",
"interface": 1
}
],
"routes": []
}
}`, ver, vethName, peerName, hostIfname, containerIfname, containerNs.Path(), containerIP.String())
args := &skel.CmdArgs{
ContainerID: "dummy",
Netns: containerNs.Path(),
IfName: containerIfname,
StdinData: []byte(conf),
}
Expect(hostNs.Do(func(_ ns.NetNS) error {
defer GinkgoRecover()
r, out, err := testutils.CmdAdd(containerNs.Path(), args.ContainerID, "", []byte(conf), func() error { return cmdAdd(args) })
Expect(err).NotTo(HaveOccurred(), string(out))
result, err := types100.GetResult(r)
Expect(err).NotTo(HaveOccurred())
Expect(result.Interfaces).To(HaveLen(5))
Expect(result.Interfaces[4].Name).To(Equal(ifbDeviceName))
Expect(result.Interfaces[4].Sandbox).To(Equal(""))
ifbLink, err := netlink.LinkByName(ifbDeviceName)
Expect(err).NotTo(HaveOccurred())
Expect(ifbLink.Attrs().MTU).To(Equal(hostIfaceMTU))
qdiscs, err := netlink.QdiscList(ifbLink)
Expect(err).NotTo(HaveOccurred())
Expect(qdiscs).To(HaveLen(1))
Expect(qdiscs[0].Attrs().LinkIndex).To(Equal(ifbLink.Attrs().Index))
Expect(qdiscs[0]).To(BeAssignableToTypeOf(&netlink.Htb{}))
Expect(qdiscs[0].(*netlink.Htb).Defcls).To(Equal(uint32(ShapedClassMinorID)))
classes, err := netlink.ClassList(ifbLink, qdiscs[0].Attrs().Handle)
Expect(err).NotTo(HaveOccurred())
Expect(classes).To(HaveLen(2))
// Uncapped class
Expect(classes[0]).To(BeAssignableToTypeOf(&netlink.HtbClass{}))
Expect(classes[0].(*netlink.HtbClass).Handle).To(Equal(netlink.MakeHandle(1, 1)))
Expect(classes[0].(*netlink.HtbClass).Rate).To(Equal(UncappedRate))
Expect(classes[0].(*netlink.HtbClass).Buffer).To(Equal(uint32(0)))
Expect(classes[0].(*netlink.HtbClass).Ceil).To(Equal(UncappedRate))
Expect(classes[0].(*netlink.HtbClass).Cbuffer).To(Equal(uint32(0)))
// Class with traffic shapping settings
Expect(classes[1]).To(BeAssignableToTypeOf(&netlink.HtbClass{}))
Expect(classes[1].(*netlink.HtbClass).Handle).To(Equal(netlink.MakeHandle(1, uint16(qdiscs[0].(*netlink.Htb).Defcls))))
Expect(classes[1].(*netlink.HtbClass).Rate).To(Equal(uint64(2)))
// Expect(classes[1].(*netlink.HtbClass).Buffer).To(Equal(uint32(7812500)))
Expect(classes[1].(*netlink.HtbClass).Ceil).To(Equal(uint64(4)))
// Expect(classes[1].(*netlink.HtbClass).Cbuffer).To(Equal(uint32(0)))
// Since we do not exclude anything from egress traffic shapping, we should not find any filter
filters, err := netlink.FilterList(ifbLink, qdiscs[0].Attrs().Handle)
Expect(err).NotTo(HaveOccurred())
Expect(filters).To(BeEmpty())
hostVethLink, err := netlink.LinkByName(hostIfname)
Expect(err).NotTo(HaveOccurred())
qdiscFilters, err := netlink.FilterList(hostVethLink, netlink.MakeHandle(0xffff, 0))
Expect(err).NotTo(HaveOccurred())
Expect(qdiscFilters).To(HaveLen(1))
Expect(qdiscFilters[0].(*netlink.U32).Actions[0].(*netlink.MirredAction).Ifindex).To(Equal(ifbLink.Attrs().Index))
return nil
})).To(Succeed())
Expect(hostNs.Do(func(_ ns.NetNS) error {
defer GinkgoRecover()
vethLink, err := netlink.LinkByName(hostIfname)
Expect(err).NotTo(HaveOccurred())
qdiscs, err := netlink.QdiscList(vethLink)
Expect(err).NotTo(HaveOccurred())
Expect(qdiscs).To(HaveLen(2))
Expect(qdiscs[0].Attrs().LinkIndex).To(Equal(vethLink.Attrs().Index))
Expect(qdiscs[0]).To(BeAssignableToTypeOf(&netlink.Htb{}))
Expect(qdiscs[0].(*netlink.Htb).Defcls).To(Equal(uint32(ShapedClassMinorID)))
classes, err := netlink.ClassList(vethLink, qdiscs[0].Attrs().Handle)
Expect(err).NotTo(HaveOccurred())
Expect(classes).To(HaveLen(2))
// Uncapped class
Expect(classes[0]).To(BeAssignableToTypeOf(&netlink.HtbClass{}))
Expect(classes[0].(*netlink.HtbClass).Handle).To(Equal(netlink.MakeHandle(1, 1)))
Expect(classes[0].(*netlink.HtbClass).Rate).To(Equal(UncappedRate))
Expect(classes[0].(*netlink.HtbClass).Buffer).To(Equal(uint32(0)))
Expect(classes[0].(*netlink.HtbClass).Ceil).To(Equal(UncappedRate))
Expect(classes[0].(*netlink.HtbClass).Cbuffer).To(Equal(uint32(0)))
// Class with traffic shapping settings
Expect(classes[1]).To(BeAssignableToTypeOf(&netlink.HtbClass{}))
Expect(classes[1].(*netlink.HtbClass).Handle).To(Equal(netlink.MakeHandle(1, uint16(qdiscs[0].(*netlink.Htb).Defcls))))
Expect(classes[1].(*netlink.HtbClass).Rate).To(Equal(uint64(1)))
// Expect(classes[1].(*netlink.HtbClass).Buffer).To(Equal(uint32(15625000)))
Expect(classes[1].(*netlink.HtbClass).Ceil).To(Equal(uint64(2)))
// Expect(classes[1].(*netlink.HtbClass).Cbuffer).To(Equal(uint32(0)))
// Since we do not exclude anything from ingress traffic shapping, we should not find any filter
filters, err := netlink.FilterList(vethLink, qdiscs[0].Attrs().Handle)
Expect(err).NotTo(HaveOccurred())
Expect(filters).To(BeEmpty())
return nil
})).To(Succeed())
})
It(fmt.Sprintf("[%s] should fail when container interface has no veth peer", ver), func() {
// create a macvlan device to be container interface
macvlanContainerIfname := "container-macv"
createMacvlan(containerNs, containerIfname, macvlanContainerIfname)
conf := fmt.Sprintf(`{
"cniVersion": "%s",
"name": "cni-plugin-bandwidth-test",
"type": "bandwidth",
"ingressRate": 8,
"ingressBurst": 8,
"egressRate": 16,
"egressBurst": 8,
"prevResult": {
"interfaces": [
{
"name": "%s",
"sandbox": ""
},
{
"name": "%s",
"sandbox": "%s"
}
],
"ips": [
{
"version": "4",
"address": "%s/24",
"gateway": "10.0.0.1",
"interface": 1
}
],
"routes": []
}
}`, ver, hostIfname, macvlanContainerIfname, containerNs.Path(), containerIP.String())
args := &skel.CmdArgs{
ContainerID: "dummy",
Netns: containerNs.Path(),
IfName: macvlanContainerIfname,
StdinData: []byte(conf),
}
Expect(hostNs.Do(func(_ ns.NetNS) error {
defer GinkgoRecover()
_, _, err := testutils.CmdAdd(containerNs.Path(), args.ContainerID, "", []byte(conf), func() error { return cmdAdd(args) })
Expect(err).To(HaveOccurred())
return nil
})).To(Succeed())
})
It(fmt.Sprintf("[%s] should fail when preResult has no interfaces", ver), func() {
conf := fmt.Sprintf(`{
"cniVersion": "%s",
"name": "cni-plugin-bandwidth-test",
"type": "bandwidth",
"ingressRate": 8,
"ingressBurst": 8,
"egressRate": 16,
"egressBurst": 8,
"prevResult": {
"interfaces": [],
"ips": [],
"routes": []
}
}`, ver)
args := &skel.CmdArgs{
ContainerID: "dummy",
Netns: containerNs.Path(),
IfName: "eth0",
StdinData: []byte(conf),
}
Expect(hostNs.Do(func(_ ns.NetNS) error {
defer GinkgoRecover()
_, _, err := testutils.CmdAdd(containerNs.Path(), args.ContainerID, "", []byte(conf), func() error { return cmdAdd(args) })
Expect(err).To(HaveOccurred())
return nil
})).To(Succeed())
})
It(fmt.Sprintf("[%s] should fail when veth peer of container interface does not match any of host interfaces in preResult", ver), func() {
// fake a non-exist host interface name
fakeHostIfname := fmt.Sprintf("%s-fake", hostIfname)
conf := fmt.Sprintf(`{
"cniVersion": "%s",
"name": "cni-plugin-bandwidth-test",
"type": "bandwidth",
"ingressRate": 8,
"ingressBurst": 8,
"egressRate": 16,
"egressBurst": 8,
"prevResult": {
"interfaces": [
{
"name": "%s",
"sandbox": ""
},
{
"name": "%s",
"sandbox": "%s"
}
],
"ips": [
{
"version": "4",
"address": "%s/24",
"gateway": "10.0.0.1",
"interface": 1
}
],
"routes": []
}
}`, ver, fakeHostIfname, containerIfname, containerNs.Path(), containerIP.String())
args := &skel.CmdArgs{
ContainerID: "dummy",
Netns: containerNs.Path(),
IfName: containerIfname,
StdinData: []byte(conf),
}
Expect(hostNs.Do(func(_ ns.NetNS) error {
defer GinkgoRecover()
_, _, err := testutils.CmdAdd(containerNs.Path(), args.ContainerID, "", []byte(conf), func() error { return cmdAdd(args) })
Expect(err).To(HaveOccurred())
return nil
})).To(Succeed())
})
})
Describe(fmt.Sprintf("[%s] QoS effective", ver), func() {
Context(fmt.Sprintf("[%s] when chaining bandwidth plugin with PTP", ver), func() {
var ptpConf string
var rateInBits uint64
var burstInBits uint64
var packetInBytes int
var containerWithoutQoSNS ns.NetNS
var containerWithQoSNS ns.NetNS
var portServerWithQoS int
var portServerWithoutQoS int
var containerWithQoSRes types.Result
var containerWithoutQoSRes types.Result
var echoServerWithQoS *gexec.Session
var echoServerWithoutQoS *gexec.Session
var dataDir string
BeforeEach(func() {
rateInBytes := 1000
rateInBits = uint64(rateInBytes * 8)
burstInBits = rateInBits * 2
// NOTE: Traffic shapping is not that precise at low rates, would be better to use higher rates + simple time+netcat for data transfer, rather than the provided
// client/server bin (limited to small amount of data)
packetInBytes = rateInBytes * 3
var err error
dataDir, err = os.MkdirTemp("", "bandwidth_linux_test")
Expect(err).NotTo(HaveOccurred())
ptpConf = fmt.Sprintf(`{
"cniVersion": "%s",
"name": "myBWnet",
"type": "ptp",
"ipMasq": true,
"mtu": 512,
"ipam": {
"type": "host-local",
"subnet": "10.1.2.0/24",
"dataDir": "%s"
}
}`, ver, dataDir)
const (
containerWithQoSIFName = "ptp0"
containerWithoutQoSIFName = "ptp1"
)
containerWithQoSNS, err = testutils.NewNS()
Expect(err).NotTo(HaveOccurred())
containerWithoutQoSNS, err = testutils.NewNS()
Expect(err).NotTo(HaveOccurred())
By("create two containers, and use the bandwidth plugin on one of them")
Expect(hostNs.Do(func(ns.NetNS) error {
defer GinkgoRecover()
containerWithQoSRes, _, err = testutils.CmdAdd(containerWithQoSNS.Path(), "dummy", containerWithQoSIFName, []byte(ptpConf), func() error {
r, err := invoke.DelegateAdd(context.TODO(), "ptp", []byte(ptpConf), nil)
Expect(err).NotTo(HaveOccurred())
Expect(r.Print()).To(Succeed())
return err
})
Expect(err).NotTo(HaveOccurred())
containerWithoutQoSRes, _, err = testutils.CmdAdd(containerWithoutQoSNS.Path(), "dummy2", containerWithoutQoSIFName, []byte(ptpConf), func() error {
r, err := invoke.DelegateAdd(context.TODO(), "ptp", []byte(ptpConf), nil)
Expect(err).NotTo(HaveOccurred())
Expect(r.Print()).To(Succeed())
return err
})
Expect(err).NotTo(HaveOccurred())
containerWithQoSResult, err := types100.GetResult(containerWithQoSRes)
Expect(err).NotTo(HaveOccurred())
bandwidthPluginConf := &PluginConf{}
err = json.Unmarshal([]byte(ptpConf), &bandwidthPluginConf)
Expect(err).NotTo(HaveOccurred())
bandwidthPluginConf.RuntimeConfig.Bandwidth = &BandwidthEntry{
IngressBurst: burstInBits,
IngressRate: rateInBits,
EgressBurst: burstInBits,
EgressRate: rateInBits,
}
bandwidthPluginConf.Type = "bandwidth"
newConfBytes, err := buildOneConfig(ver, bandwidthPluginConf, containerWithQoSResult)
Expect(err).NotTo(HaveOccurred())
args := &skel.CmdArgs{
ContainerID: "dummy3",
Netns: containerWithQoSNS.Path(),
IfName: containerWithQoSIFName,
StdinData: newConfBytes,
}
result, out, err := testutils.CmdAdd(containerWithQoSNS.Path(), args.ContainerID, "", newConfBytes, func() error { return cmdAdd(args) })
Expect(err).NotTo(HaveOccurred(), string(out))
if testutils.SpecVersionHasCHECK(ver) {
// Do CNI Check
checkConf := &PluginConf{}
err = json.Unmarshal([]byte(ptpConf), &checkConf)
Expect(err).NotTo(HaveOccurred())
checkConf.RuntimeConfig.Bandwidth = &BandwidthEntry{
IngressBurst: burstInBits,
IngressRate: rateInBits,
EgressBurst: burstInBits,
EgressRate: rateInBits,
}
checkConf.Type = "bandwidth"
newCheckBytes, err := buildOneConfig(ver, checkConf, result)
Expect(err).NotTo(HaveOccurred())
args = &skel.CmdArgs{
ContainerID: "dummy3",
Netns: containerWithQoSNS.Path(),
IfName: containerWithQoSIFName,
StdinData: newCheckBytes,
}
err = testutils.CmdCheck(containerWithQoSNS.Path(), args.ContainerID, "", func() error { return cmdCheck(args) })
Expect(err).NotTo(HaveOccurred())
}
return nil
})).To(Succeed())
By("starting a tcp server on both containers")
portServerWithQoS, echoServerWithQoS = startEchoServerInNamespace(containerWithQoSNS)
portServerWithoutQoS, echoServerWithoutQoS = startEchoServerInNamespace(containerWithoutQoSNS)
})
AfterEach(func() {
Expect(os.RemoveAll(dataDir)).To(Succeed())
Expect(containerWithQoSNS.Close()).To(Succeed())
Expect(testutils.UnmountNS(containerWithQoSNS)).To(Succeed())
Expect(containerWithoutQoSNS.Close()).To(Succeed())
Expect(testutils.UnmountNS(containerWithoutQoSNS)).To(Succeed())
if echoServerWithoutQoS != nil {
echoServerWithoutQoS.Kill()
}
if echoServerWithQoS != nil {
echoServerWithQoS.Kill()
}
})
It("limits ingress traffic on veth device", func() {
var runtimeWithLimit time.Duration
var runtimeWithoutLimit time.Duration
By("gather timing statistics about both containers")
By("sending tcp traffic to the container that has traffic shaped", func() {
start := time.Now()
result, err := types100.GetResult(containerWithQoSRes)
Expect(err).NotTo(HaveOccurred())
makeTCPClientInNS(hostNs.Path(), result.IPs[0].Address.IP.String(), portServerWithQoS, packetInBytes)
end := time.Now()
runtimeWithLimit = end.Sub(start)
log.Printf("Elapsed with qos %.2f", runtimeWithLimit.Seconds())
})
By("sending tcp traffic to the container that does not have traffic shaped", func() {
start := time.Now()
result, err := types100.GetResult(containerWithoutQoSRes)
Expect(err).NotTo(HaveOccurred())
makeTCPClientInNS(hostNs.Path(), result.IPs[0].Address.IP.String(), portServerWithoutQoS, packetInBytes)
end := time.Now()
runtimeWithoutLimit = end.Sub(start)
log.Printf("Elapsed without qos %.2f", runtimeWithoutLimit.Seconds())
})
Expect(runtimeWithLimit).To(BeNumerically(">", runtimeWithoutLimit+1000*time.Millisecond))
})
})
})
Context(fmt.Sprintf("[%s] when chaining bandwidth plugin with PTP and excluding specific subnets from traffic", ver), func() {
var ptpConf string
var rateInBits uint64
var burstInBits uint64
var packetInBytes int
var containerWithoutQoSNS ns.NetNS
var containerWithQoSNS ns.NetNS
var portServerWithQoS int
var portServerWithoutQoS int
var containerWithQoSRes types.Result
var containerWithoutQoSRes types.Result
var echoServerWithQoS *gexec.Session
var echoServerWithoutQoS *gexec.Session
var dataDir string
BeforeEach(func() {
rateInBytes := 1000
rateInBits = uint64(rateInBytes * 8)
burstInBits = rateInBits * 2
unshapedSubnets := []string{"10.1.2.0/24"}
// NOTE: Traffic shapping is not that precise at low rates, would be better to use higher rates + simple time+netcat for data transfer, rather than the provided
// client/server bin (limited to small amount of data)
packetInBytes = rateInBytes * 3
var err error
dataDir, err = os.MkdirTemp("", "bandwidth_linux_test")
Expect(err).NotTo(HaveOccurred())
ptpConf = fmt.Sprintf(`{
"cniVersion": "%s",
"name": "myBWnet",
"type": "ptp",
"ipMasq": true,
"mtu": 512,
"ipam": {
"type": "host-local",
"subnet": "10.1.2.0/24",
"dataDir": "%s"
}
}`, ver, dataDir)
const (
containerWithQoSIFName = "ptp0"
containerWithoutQoSIFName = "ptp1"
)
containerWithQoSNS, err = testutils.NewNS()
Expect(err).NotTo(HaveOccurred())
containerWithoutQoSNS, err = testutils.NewNS()
Expect(err).NotTo(HaveOccurred())
By("create two containers, and use the bandwidth plugin on one of them")
Expect(hostNs.Do(func(ns.NetNS) error {
defer GinkgoRecover()
containerWithQoSRes, _, err = testutils.CmdAdd(containerWithQoSNS.Path(), "dummy", containerWithQoSIFName, []byte(ptpConf), func() error {
r, err := invoke.DelegateAdd(context.TODO(), "ptp", []byte(ptpConf), nil)
Expect(err).NotTo(HaveOccurred())
Expect(r.Print()).To(Succeed())
return err
})
Expect(err).NotTo(HaveOccurred())
containerWithoutQoSRes, _, err = testutils.CmdAdd(containerWithoutQoSNS.Path(), "dummy2", containerWithoutQoSIFName, []byte(ptpConf), func() error {
r, err := invoke.DelegateAdd(context.TODO(), "ptp", []byte(ptpConf), nil)
Expect(err).NotTo(HaveOccurred())
Expect(r.Print()).To(Succeed())
return err
})
Expect(err).NotTo(HaveOccurred())
containerWithQoSResult, err := types100.GetResult(containerWithQoSRes)
Expect(err).NotTo(HaveOccurred())
bandwidthPluginConf := &PluginConf{}
err = json.Unmarshal([]byte(ptpConf), &bandwidthPluginConf)
Expect(err).NotTo(HaveOccurred())
bandwidthPluginConf.RuntimeConfig.Bandwidth = &BandwidthEntry{
IngressBurst: burstInBits,
IngressRate: rateInBits,
EgressBurst: burstInBits,
EgressRate: rateInBits,
UnshapedSubnets: unshapedSubnets,
}
bandwidthPluginConf.Type = "bandwidth"
newConfBytes, err := buildOneConfig(ver, bandwidthPluginConf, containerWithQoSResult)
Expect(err).NotTo(HaveOccurred())
args := &skel.CmdArgs{
ContainerID: "dummy3",
Netns: containerWithQoSNS.Path(),
IfName: containerWithQoSIFName,
StdinData: newConfBytes,
}
result, out, err := testutils.CmdAdd(containerWithQoSNS.Path(), args.ContainerID, "", newConfBytes, func() error { return cmdAdd(args) })
Expect(err).NotTo(HaveOccurred(), string(out))
if testutils.SpecVersionHasCHECK(ver) {
// Do CNI Check
checkConf := &PluginConf{}
err = json.Unmarshal([]byte(ptpConf), &checkConf)
Expect(err).NotTo(HaveOccurred())
checkConf.RuntimeConfig.Bandwidth = &BandwidthEntry{
IngressBurst: burstInBits,
IngressRate: rateInBits,
EgressBurst: burstInBits,
EgressRate: rateInBits,
UnshapedSubnets: unshapedSubnets,
}
checkConf.Type = "bandwidth"
newCheckBytes, err := buildOneConfig(ver, checkConf, result)
Expect(err).NotTo(HaveOccurred())
args = &skel.CmdArgs{
ContainerID: "dummy3",
Netns: containerWithQoSNS.Path(),
IfName: containerWithQoSIFName,
StdinData: newCheckBytes,
}
err = testutils.CmdCheck(containerWithQoSNS.Path(), args.ContainerID, "", func() error { return cmdCheck(args) })
Expect(err).NotTo(HaveOccurred())
}
return nil
})).To(Succeed())
By("starting a tcp server on both containers")
portServerWithQoS, echoServerWithQoS = startEchoServerInNamespace(containerWithQoSNS)
portServerWithoutQoS, echoServerWithoutQoS = startEchoServerInNamespace(containerWithoutQoSNS)
})
AfterEach(func() {
Expect(os.RemoveAll(dataDir)).To(Succeed())
Expect(containerWithQoSNS.Close()).To(Succeed())
Expect(testutils.UnmountNS(containerWithQoSNS)).To(Succeed())
Expect(containerWithoutQoSNS.Close()).To(Succeed())
Expect(testutils.UnmountNS(containerWithoutQoSNS)).To(Succeed())
if echoServerWithoutQoS != nil {
echoServerWithoutQoS.Kill()
}
if echoServerWithQoS != nil {
echoServerWithQoS.Kill()
}
})
It("does not limits ingress traffic on veth device coming from 10.1.2.0/24", func() {
var runtimeWithLimit time.Duration
var runtimeWithoutLimit time.Duration
By("gather timing statistics about both containers")
By("sending tcp traffic to the container that has traffic shaped", func() {
start := time.Now()
result, err := types100.GetResult(containerWithQoSRes)
Expect(err).NotTo(HaveOccurred())
makeTCPClientInNS(hostNs.Path(), result.IPs[0].Address.IP.String(), portServerWithQoS, packetInBytes)
end := time.Now()
runtimeWithLimit = end.Sub(start)
log.Printf("Elapsed with qos %.2f", runtimeWithLimit.Seconds())
})
By("sending tcp traffic to the container that does not have traffic shaped", func() {
start := time.Now()
result, err := types100.GetResult(containerWithoutQoSRes)
Expect(err).NotTo(HaveOccurred())
makeTCPClientInNS(hostNs.Path(), result.IPs[0].Address.IP.String(), portServerWithoutQoS, packetInBytes)
end := time.Now()
runtimeWithoutLimit = end.Sub(start)
log.Printf("Elapsed without qos %.2f", runtimeWithoutLimit.Seconds())
})
Expect(runtimeWithLimit - runtimeWithoutLimit).To(BeNumerically("<", 100*time.Millisecond))
})
})
Context(fmt.Sprintf("[%s] when chaining bandwidth plugin with PTP and only including specific subnets in traffic shapping (not including the main ns one)", ver), func() {
var ptpConf string
var rateInBits uint64
var burstInBits uint64
var packetInBytes int
var containerWithoutQoSNS ns.NetNS
var containerWithQoSNS ns.NetNS
var portServerWithQoS int
var portServerWithoutQoS int
var containerWithQoSRes types.Result
var containerWithoutQoSRes types.Result
var echoServerWithQoS *gexec.Session
var echoServerWithoutQoS *gexec.Session
var dataDir string
BeforeEach(func() {
rateInBytes := 1000
rateInBits = uint64(rateInBytes * 8)
burstInBits = rateInBits * 2
shapedSubnets := []string{"10.2.2.0/24"}
// NOTE: Traffic shapping is not that precise at low rates, would be better to use higher rates + simple time+netcat for data transfer, rather than the provided
// client/server bin (limited to small amount of data)
packetInBytes = rateInBytes * 3
var err error
dataDir, err = os.MkdirTemp("", "bandwidth_linux_test")
Expect(err).NotTo(HaveOccurred())
ptpConf = fmt.Sprintf(`{
"cniVersion": "%s",
"name": "myBWnet",
"type": "ptp",
"ipMasq": true,
"mtu": 512,
"ipam": {
"type": "host-local",
"subnet": "10.1.2.0/24",
"dataDir": "%s"
}
}`, ver, dataDir)
const (
containerWithQoSIFName = "ptp0"
containerWithoutQoSIFName = "ptp1"
)
containerWithQoSNS, err = testutils.NewNS()
Expect(err).NotTo(HaveOccurred())
containerWithoutQoSNS, err = testutils.NewNS()
Expect(err).NotTo(HaveOccurred())
By("create two containers, and use the bandwidth plugin on one of them")
Expect(hostNs.Do(func(ns.NetNS) error {
defer GinkgoRecover()
containerWithQoSRes, _, err = testutils.CmdAdd(containerWithQoSNS.Path(), "dummy", containerWithQoSIFName, []byte(ptpConf), func() error {
r, err := invoke.DelegateAdd(context.TODO(), "ptp", []byte(ptpConf), nil)
Expect(err).NotTo(HaveOccurred())
Expect(r.Print()).To(Succeed())
return err
})
Expect(err).NotTo(HaveOccurred())
containerWithoutQoSRes, _, err = testutils.CmdAdd(containerWithoutQoSNS.Path(), "dummy2", containerWithoutQoSIFName, []byte(ptpConf), func() error {
r, err := invoke.DelegateAdd(context.TODO(), "ptp", []byte(ptpConf), nil)
Expect(err).NotTo(HaveOccurred())
Expect(r.Print()).To(Succeed())
return err
})
Expect(err).NotTo(HaveOccurred())
containerWithQoSResult, err := types100.GetResult(containerWithQoSRes)
Expect(err).NotTo(HaveOccurred())
bandwidthPluginConf := &PluginConf{}
err = json.Unmarshal([]byte(ptpConf), &bandwidthPluginConf)
Expect(err).NotTo(HaveOccurred())
bandwidthPluginConf.RuntimeConfig.Bandwidth = &BandwidthEntry{
IngressBurst: burstInBits,
IngressRate: rateInBits,
EgressBurst: burstInBits,
EgressRate: rateInBits,
ShapedSubnets: shapedSubnets,
}
bandwidthPluginConf.Type = "bandwidth"
newConfBytes, err := buildOneConfig(ver, bandwidthPluginConf, containerWithQoSResult)
Expect(err).NotTo(HaveOccurred())
args := &skel.CmdArgs{
ContainerID: "dummy3",
Netns: containerWithQoSNS.Path(),
IfName: containerWithQoSIFName,
StdinData: newConfBytes,
}
result, out, err := testutils.CmdAdd(containerWithQoSNS.Path(), args.ContainerID, "", newConfBytes, func() error { return cmdAdd(args) })
Expect(err).NotTo(HaveOccurred(), string(out))
if testutils.SpecVersionHasCHECK(ver) {
// Do CNI Check
checkConf := &PluginConf{}
err = json.Unmarshal([]byte(ptpConf), &checkConf)
Expect(err).NotTo(HaveOccurred())
checkConf.RuntimeConfig.Bandwidth = &BandwidthEntry{
IngressBurst: burstInBits,
IngressRate: rateInBits,
EgressBurst: burstInBits,
EgressRate: rateInBits,
ShapedSubnets: shapedSubnets,
}
checkConf.Type = "bandwidth"
newCheckBytes, err := buildOneConfig(ver, checkConf, result)
Expect(err).NotTo(HaveOccurred())
args = &skel.CmdArgs{
ContainerID: "dummy3",
Netns: containerWithQoSNS.Path(),
IfName: containerWithQoSIFName,
StdinData: newCheckBytes,
}
err = testutils.CmdCheck(containerWithQoSNS.Path(), args.ContainerID, "", func() error { return cmdCheck(args) })
Expect(err).NotTo(HaveOccurred())
}
return nil
})).To(Succeed())
By("starting a tcp server on both containers")
portServerWithQoS, echoServerWithQoS = startEchoServerInNamespace(containerWithQoSNS)
portServerWithoutQoS, echoServerWithoutQoS = startEchoServerInNamespace(containerWithoutQoSNS)
})
AfterEach(func() {
Expect(os.RemoveAll(dataDir)).To(Succeed())
Expect(containerWithQoSNS.Close()).To(Succeed())
Expect(testutils.UnmountNS(containerWithQoSNS)).To(Succeed())
Expect(containerWithoutQoSNS.Close()).To(Succeed())
Expect(testutils.UnmountNS(containerWithoutQoSNS)).To(Succeed())
if echoServerWithoutQoS != nil {
echoServerWithoutQoS.Kill()
}
if echoServerWithQoS != nil {
echoServerWithQoS.Kill()
}
})
It("does not limit ingress traffic on veth device coming from non included subnets", func() {
var runtimeWithLimit time.Duration
var runtimeWithoutLimit time.Duration
By("gather timing statistics about both containers")
By("sending tcp traffic to the container that has traffic shaped", func() {
start := time.Now()
result, err := types100.GetResult(containerWithQoSRes)
Expect(err).NotTo(HaveOccurred())
makeTCPClientInNS(hostNs.Path(), result.IPs[0].Address.IP.String(), portServerWithQoS, packetInBytes)
end := time.Now()
runtimeWithLimit = end.Sub(start)
log.Printf("Elapsed with qos %.2f", runtimeWithLimit.Seconds())
})
By("sending tcp traffic to the container that does not have traffic shaped", func() {
start := time.Now()
result, err := types100.GetResult(containerWithoutQoSRes)
Expect(err).NotTo(HaveOccurred())
makeTCPClientInNS(hostNs.Path(), result.IPs[0].Address.IP.String(), portServerWithoutQoS, packetInBytes)
end := time.Now()
runtimeWithoutLimit = end.Sub(start)
log.Printf("Elapsed without qos %.2f", runtimeWithoutLimit.Seconds())
})
Expect(runtimeWithLimit - runtimeWithoutLimit).To(BeNumerically("<", 100*time.Millisecond))
})
})
Context(fmt.Sprintf("[%s] when chaining bandwidth plugin with PTP and only including specific subnets in traffic shapping (including the main ns one)", ver), func() {
var ptpConf string
var rateInBits uint64
var burstInBits uint64
var packetInBytes int
var containerWithoutQoSNS ns.NetNS
var containerWithQoSNS ns.NetNS
var portServerWithQoS int
var portServerWithoutQoS int
var containerWithQoSRes types.Result
var containerWithoutQoSRes types.Result
var echoServerWithQoS *gexec.Session
var echoServerWithoutQoS *gexec.Session
var dataDir string
BeforeEach(func() {
rateInBytes := 1000
rateInBits = uint64(rateInBytes * 8)
burstInBits = rateInBits * 2
shapedSubnets := []string{"10.1.2.1/32"}
// NOTE: Traffic shapping is not that precise at low rates, would be better to use higher rates + simple time+netcat for data transfer, rather than the provided
// client/server bin (limited to small amount of data)
packetInBytes = rateInBytes * 3
var err error
dataDir, err = os.MkdirTemp("", "bandwidth_linux_test")
Expect(err).NotTo(HaveOccurred())
ptpConf = fmt.Sprintf(`{
"cniVersion": "%s",
"name": "myBWnet",
"type": "ptp",
"ipMasq": true,
"mtu": 512,
"ipam": {
"type": "host-local",
"subnet": "10.1.2.0/24",
"dataDir": "%s"
}
}`, ver, dataDir)
const (
containerWithQoSIFName = "ptp0"
containerWithoutQoSIFName = "ptp1"
)
containerWithQoSNS, err = testutils.NewNS()
Expect(err).NotTo(HaveOccurred())
containerWithoutQoSNS, err = testutils.NewNS()
Expect(err).NotTo(HaveOccurred())
By("create two containers, and use the bandwidth plugin on one of them")
Expect(hostNs.Do(func(ns.NetNS) error {
defer GinkgoRecover()
containerWithQoSRes, _, err = testutils.CmdAdd(containerWithQoSNS.Path(), "dummy", containerWithQoSIFName, []byte(ptpConf), func() error {
r, err := invoke.DelegateAdd(context.TODO(), "ptp", []byte(ptpConf), nil)
Expect(err).NotTo(HaveOccurred())
Expect(r.Print()).To(Succeed())
return err
})
Expect(err).NotTo(HaveOccurred())
containerWithoutQoSRes, _, err = testutils.CmdAdd(containerWithoutQoSNS.Path(), "dummy2", containerWithoutQoSIFName, []byte(ptpConf), func() error {
r, err := invoke.DelegateAdd(context.TODO(), "ptp", []byte(ptpConf), nil)
Expect(err).NotTo(HaveOccurred())
Expect(r.Print()).To(Succeed())
return err
})
Expect(err).NotTo(HaveOccurred())
containerWithQoSResult, err := types100.GetResult(containerWithQoSRes)
Expect(err).NotTo(HaveOccurred())
bandwidthPluginConf := &PluginConf{}
err = json.Unmarshal([]byte(ptpConf), &bandwidthPluginConf)
Expect(err).NotTo(HaveOccurred())
bandwidthPluginConf.RuntimeConfig.Bandwidth = &BandwidthEntry{
IngressBurst: burstInBits,
IngressRate: rateInBits,
EgressBurst: burstInBits,
EgressRate: rateInBits,
ShapedSubnets: shapedSubnets,
}
bandwidthPluginConf.Type = "bandwidth"
newConfBytes, err := buildOneConfig(ver, bandwidthPluginConf, containerWithQoSResult)
Expect(err).NotTo(HaveOccurred())
args := &skel.CmdArgs{
ContainerID: "dummy3",
Netns: containerWithQoSNS.Path(),
IfName: containerWithQoSIFName,
StdinData: newConfBytes,
}
result, out, err := testutils.CmdAdd(containerWithQoSNS.Path(), args.ContainerID, "", newConfBytes, func() error { return cmdAdd(args) })
Expect(err).NotTo(HaveOccurred(), string(out))
if testutils.SpecVersionHasCHECK(ver) {
// Do CNI Check
checkConf := &PluginConf{}
err = json.Unmarshal([]byte(ptpConf), &checkConf)
Expect(err).NotTo(HaveOccurred())
checkConf.RuntimeConfig.Bandwidth = &BandwidthEntry{
IngressBurst: burstInBits,
IngressRate: rateInBits,
EgressBurst: burstInBits,
EgressRate: rateInBits,
ShapedSubnets: shapedSubnets,
}
checkConf.Type = "bandwidth"
newCheckBytes, err := buildOneConfig(ver, checkConf, result)
Expect(err).NotTo(HaveOccurred())
args = &skel.CmdArgs{
ContainerID: "dummy3",
Netns: containerWithQoSNS.Path(),
IfName: containerWithQoSIFName,
StdinData: newCheckBytes,
}
err = testutils.CmdCheck(containerWithQoSNS.Path(), args.ContainerID, "", func() error { return cmdCheck(args) })
Expect(err).NotTo(HaveOccurred())
}
return nil
})).To(Succeed())
By("starting a tcp server on both containers")
portServerWithQoS, echoServerWithQoS = startEchoServerInNamespace(containerWithQoSNS)
portServerWithoutQoS, echoServerWithoutQoS = startEchoServerInNamespace(containerWithoutQoSNS)
})
AfterEach(func() {
Expect(os.RemoveAll(dataDir)).To(Succeed())
Expect(containerWithQoSNS.Close()).To(Succeed())
Expect(testutils.UnmountNS(containerWithQoSNS)).To(Succeed())
Expect(containerWithoutQoSNS.Close()).To(Succeed())
Expect(testutils.UnmountNS(containerWithoutQoSNS)).To(Succeed())
if echoServerWithoutQoS != nil {
echoServerWithoutQoS.Kill()
}
if echoServerWithQoS != nil {
echoServerWithQoS.Kill()
}
})
It("limits ingress traffic on veth device coming from included subnets", func() {
var runtimeWithLimit time.Duration
var runtimeWithoutLimit time.Duration
By("gather timing statistics about both containers")
By("sending tcp traffic to the container that has traffic shaped", func() {
start := time.Now()
result, err := types100.GetResult(containerWithQoSRes)
Expect(err).NotTo(HaveOccurred())
makeTCPClientInNS(hostNs.Path(), result.IPs[0].Address.IP.String(), portServerWithQoS, packetInBytes)
end := time.Now()
runtimeWithLimit = end.Sub(start)
log.Printf("Elapsed with qos %.2f", runtimeWithLimit.Seconds())
})
By("sending tcp traffic to the container that does not have traffic shaped", func() {
start := time.Now()
result, err := types100.GetResult(containerWithoutQoSRes)
Expect(err).NotTo(HaveOccurred())
makeTCPClientInNS(hostNs.Path(), result.IPs[0].Address.IP.String(), portServerWithoutQoS, packetInBytes)
end := time.Now()
runtimeWithoutLimit = end.Sub(start)
log.Printf("Elapsed without qos %.2f", runtimeWithoutLimit.Seconds())
})
Expect(runtimeWithLimit).To(BeNumerically(">", runtimeWithoutLimit+1000*time.Millisecond))
})
})
}
Describe("Validating input", func() {
It("Should allow only 4GB burst rate", func() {
err := validateRateAndBurst(5000, 4*1024*1024*1024*8-16) // 2 bytes less than the max should pass
Expect(err).NotTo(HaveOccurred())
err = validateRateAndBurst(5000, 4*1024*1024*1024*8) // we're 1 bit above MaxUint32
Expect(err).To(HaveOccurred())
err = validateRateAndBurst(0, 1)
Expect(err).To(HaveOccurred())
err = validateRateAndBurst(1, 0)
Expect(err).To(HaveOccurred())
err = validateRateAndBurst(0, 0)
Expect(err).NotTo(HaveOccurred())
})
It("Should fail if both ShapedSubnets and UnshapedSubnets are specified", func() {
err := validateSubnets([]string{"10.0.0.0/8"}, []string{"192.168.0.0/24"})
Expect(err).To(HaveOccurred())
})
It("Should fail if specified UnshapedSubnets are not valid CIDRs", func() {
err := validateSubnets([]string{"10.0.0.0/8", "hello"}, []string{})
Expect(err).To(HaveOccurred())
})
It("Should fail if specified ShapedSubnets are not valid CIDRs", func() {
err := validateSubnets([]string{}, []string{"10.0.0.0/8", "hello"})
Expect(err).To(HaveOccurred())
})
})
})
View Git Blame Copy Permalink