ITQC/containernetworking-plugins
24
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 34 Wiki Activity

bandwidth: possibility to specify shaped subnets or to exclude some from shaping

Browse Source 
Signed-off-by: Raphael <oOraph@users.noreply.github.com>
This commit is contained in:
Raphael 2023-07-27 10:02:57 +02:00 committed by Tomofumi Hayashi
parent 52da39d3aa
commit ab0b386b4e
3 changed files with 1092 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1021
plugins/meta/bandwidth/bandwidth_linux_test.go
View File

File diff suppressed because it is too large Load Diff

71
plugins/meta/bandwidth/ifb_creator.go
View File

@ -27,9 +27,10 @@ import (
) )
const ( const (
latencyInMillis = 25 latencyInMillis = 25
UncappedRate uint64 = 100_000_000_000 UncappedRate uint64 = 100_000_000_000
DefaultClassMinorID = 48 ShapedClassMinorID uint16 = 48
UnShapedClassMinorID uint16 = 1
) )
func CreateIfb(ifbDeviceName string, mtu int, qlen int) error { func CreateIfb(ifbDeviceName string, mtu int, qlen int) error {
@ -60,15 +61,24 @@ func TeardownIfb(deviceName string) error {
return err return err
} }
func CreateIngressQdisc(rateInBits, burstInBits uint64, excludeSubnets []string, hostDeviceName string) error { func CreateIngressQdisc(rateInBits, burstInBits uint64, excludeSubnets []string, includeSubnets []string, hostDeviceName string) error {
hostDevice, err := netlink.LinkByName(hostDeviceName) hostDevice, err := netlink.LinkByName(hostDeviceName)
if err != nil { if err != nil {
return fmt.Errorf("get host device: %s", err) return fmt.Errorf("get host device: %s", err)
} }
return createHTB(rateInBits, burstInBits, hostDevice.Attrs().Index, excludeSubnets)
subnets := includeSubnets
exclude := false
if len(excludeSubnets) > 0 {
subnets = excludeSubnets
exclude = true
}
return createHTB(rateInBits, burstInBits, hostDevice.Attrs().Index, subnets, exclude)
} }
func CreateEgressQdisc(rateInBits, burstInBits uint64, excludeSubnets []string, hostDeviceName string, ifbDeviceName string) error { func CreateEgressQdisc(rateInBits, burstInBits uint64, excludeSubnets []string, includeSubnets []string, hostDeviceName string, ifbDeviceName string) error {
ifbDevice, err := netlink.LinkByName(ifbDeviceName) ifbDevice, err := netlink.LinkByName(ifbDeviceName)
if err != nil { if err != nil {
return fmt.Errorf("get ifb device: %s", err) return fmt.Errorf("get ifb device: %s", err)
@ -115,8 +125,16 @@ func CreateEgressQdisc(rateInBits, burstInBits uint64, excludeSubnets []string,
return fmt.Errorf("add filter: %s", err) return fmt.Errorf("add filter: %s", err)
} }
subnets := excludeSubnets
exclude := true
if len(includeSubnets) > 0 {
subnets = includeSubnets
exclude = false
}
// throttle traffic on ifb device // throttle traffic on ifb device
err = createHTB(rateInBits, burstInBits, ifbDevice.Attrs().Index, excludeSubnets) err = createHTB(rateInBits, burstInBits, ifbDevice.Attrs().Index, subnets, exclude)
if err != nil { if err != nil {
// egress from the container/netns pov = ingress from the main netns/host pov // egress from the container/netns pov = ingress from the main netns/host pov
return fmt.Errorf("create htb container egress qos rules: %s", err) return fmt.Errorf("create htb container egress qos rules: %s", err)
@ -124,9 +142,15 @@ func CreateEgressQdisc(rateInBits, burstInBits uint64, excludeSubnets []string,
return nil return nil
} }
func createHTB(rateInBits, burstInBits uint64, linkIndex int, excludeSubnets []string) error { func createHTB(rateInBits, burstInBits uint64, linkIndex int, subnets []string, excludeSubnets bool) error {
// Netlink struct fields are not clear, let's use shell // Netlink struct fields are not clear, let's use shell
defaultClassID := UnShapedClassMinorID
// If no subnets are specified, then shaping should apply to everything
if len(subnets) == 0 || excludeSubnets {
defaultClassID = ShapedClassMinorID
}
// Step 1 qdisc // Step 1 qdisc
// cmd := exec.Command("/usr/sbin/tc", "qdisc", "add", "dev", interfaceName, "root", "handle", "1:", "htb", "default", "30") // cmd := exec.Command("/usr/sbin/tc", "qdisc", "add", "dev", interfaceName, "root", "handle", "1:", "htb", "default", "30")
qdisc := &netlink.Htb{ qdisc := &netlink.Htb{
@ -135,7 +159,7 @@ func createHTB(rateInBits, burstInBits uint64, linkIndex int, excludeSubnets []s
Handle: netlink.MakeHandle(1, 0), Handle: netlink.MakeHandle(1, 0),
Parent: netlink.HANDLE_ROOT, Parent: netlink.HANDLE_ROOT,
}, },
Defcls: DefaultClassMinorID, Defcls: uint32(defaultClassID),
// No idea what these are so let's keep the default values from source code... // No idea what these are so let's keep the default values from source code...
Version: 3, Version: 3,
Rate2Quantum: 10, Rate2Quantum: 10,
@ -151,13 +175,13 @@ func createHTB(rateInBits, burstInBits uint64, linkIndex int, excludeSubnets []s
burstInBytes := burstInBits / 8 burstInBytes := burstInBits / 8
bufferInBytes := buffer(rateInBytes, uint32(burstInBytes)) bufferInBytes := buffer(rateInBytes, uint32(burstInBytes))
// The capped class for all but excluded subnets // The capped class for shaped traffic (included subnets or all but excluded subnets)
// cmd = exec.Command("/usr/sbin/tc", "class", "add", "dev", interfaceName, "parent", "1:", "classid", "1:30", "htb", "rate", // cmd = exec.Command("/usr/sbin/tc", "class", "add", "dev", interfaceName, "parent", "1:", "classid", "1:30", "htb", "rate",
// fmt.Sprintf("%d", rateInBits), "burst", fmt.Sprintf("%d", burstInBits)) // fmt.Sprintf("%d", rateInBits), "burst", fmt.Sprintf("%d", burstInBits))
defClass := &netlink.HtbClass{ shapedClass := &netlink.HtbClass{
ClassAttrs: netlink.ClassAttrs{ ClassAttrs: netlink.ClassAttrs{
LinkIndex: linkIndex, LinkIndex: linkIndex,
Handle: netlink.MakeHandle(1, DefaultClassMinorID), Handle: netlink.MakeHandle(1, ShapedClassMinorID),
Parent: netlink.MakeHandle(1, 0), Parent: netlink.MakeHandle(1, 0),
}, },
Rate: rateInBytes, Rate: rateInBytes,
@ -167,19 +191,19 @@ func createHTB(rateInBits, burstInBits uint64, linkIndex int, excludeSubnets []s
Cbuffer: bufferInBytes, Cbuffer: bufferInBytes,
} }
err = netlink.ClassAdd(defClass) err = netlink.ClassAdd(shapedClass)
if err != nil { if err != nil {
return fmt.Errorf("error while creating htb default class: %s", err) return fmt.Errorf("error while creating htb default class: %s", err)
} }
// The uncapped class for the excluded subnets // The uncapped class for non shaped traffic (either all but included subnets or excluded subnets only)
// cmd = exec.Command("/usr/sbin/tc", "class", "add", "dev", interfaceName, "parent", "1:", "classid", "1:1", "htb", // cmd = exec.Command("/usr/sbin/tc", "class", "add", "dev", interfaceName, "parent", "1:", "classid", "1:1", "htb",
// "rate", "100000000000") // "rate", "100000000000")
bigRate := UncappedRate bigRate := UncappedRate
uncappedClass := &netlink.HtbClass{ unshapedClass := &netlink.HtbClass{
ClassAttrs: netlink.ClassAttrs{ ClassAttrs: netlink.ClassAttrs{
LinkIndex: linkIndex, LinkIndex: linkIndex,
Handle: netlink.MakeHandle(1, 1), Handle: netlink.MakeHandle(1, UnShapedClassMinorID),
Parent: qdisc.Handle, Parent: qdisc.Handle,
}, },
Rate: bigRate, Rate: bigRate,
@ -187,14 +211,14 @@ func createHTB(rateInBits, burstInBits uint64, linkIndex int, excludeSubnets []s
// No need for any burst, the minimum buffer size in q_htb.c should be enough to handle the rate which // No need for any burst, the minimum buffer size in q_htb.c should be enough to handle the rate which
// is already more than enough // is already more than enough
} }
err = netlink.ClassAdd(uncappedClass) err = netlink.ClassAdd(unshapedClass)
if err != nil { if err != nil {
return fmt.Errorf("error while creating htb uncapped class: %s", err) return fmt.Errorf("error while creating htb uncapped class: %s", err)
} }
// Now add filters to redirect excluded subnets to the class 1 instead of the default one (30) // Now add filters to redirect subnets to the class 1 if excluded instead of the default one (30)
for _, subnet := range excludeSubnets { for _, subnet := range subnets {
// cmd = exec.Command("/usr/sbin/tc", "filter", "add", "dev", interfaceName, "parent", "1:", "protocol", protocol, // cmd = exec.Command("/usr/sbin/tc", "filter", "add", "dev", interfaceName, "parent", "1:", "protocol", protocol,
// "prio", "16", "u32", "match", "ip", "dst", subnet, "flowid", "1:1") // "prio", "16", "u32", "match", "ip", "dst", subnet, "flowid", "1:1")
@ -225,8 +249,6 @@ func createHTB(rateInBits, burstInBits uint64, linkIndex int, excludeSubnets []s
} }
// protocol := syscall.ETH_P_ALL
if len(maskBytes) < keepBytes { if len(maskBytes) < keepBytes {
return fmt.Errorf("error with net lib, unexpected count of bytes for ipv4 mask (%d < %d)", return fmt.Errorf("error with net lib, unexpected count of bytes for ipv4 mask (%d < %d)",
len(maskBytes), keepBytes) len(maskBytes), keepBytes)
@ -296,6 +318,11 @@ func createHTB(rateInBits, burstInBits uint64, linkIndex int, excludeSubnets []s
} }
} }
classID := shapedClass.Handle
if excludeSubnets {
classID = unshapedClass.Handle
}
tcFilter := netlink.U32{ tcFilter := netlink.U32{
FilterAttrs: netlink.FilterAttrs{ FilterAttrs: netlink.FilterAttrs{
LinkIndex: linkIndex, LinkIndex: linkIndex,
@ -303,7 +330,7 @@ func createHTB(rateInBits, burstInBits uint64, linkIndex int, excludeSubnets []s
Priority: prio, Priority: prio,
Protocol: uint16(protocol), Protocol: uint16(protocol),
}, },
ClassId: uncappedClass.Handle, ClassId: classID,
Sel: selector, Sel: selector,
} }

65
plugins/meta/bandwidth/main.go
View File

@ -40,12 +40,12 @@ const (
// BandwidthEntry corresponds to a single entry in the bandwidth argument, // BandwidthEntry corresponds to a single entry in the bandwidth argument,
// see CONVENTIONS.md // see CONVENTIONS.md
type BandwidthEntry struct { type BandwidthEntry struct {
NonShapedSubnets []string `json:"nonShapedSubnets"` // Ipv4/ipv6 subnets to be excluded from traffic shaping UnshapedSubnets []string `json:"unshapedSubnets"` // Ipv4/ipv6 subnets to be excluded from traffic shaping. UnshapedSubnets and ShapedSubnets parameters are mutually exlusive
IngressRate uint64 `json:"ingressRate"` // Bandwidth rate in bps for traffic through container. 0 for no limit. If ingressRate is set, ingressBurst must also be set ShapedSubnets []string `json:"shapedSubnets"` // Ipv4/ipv6 subnets to be included in traffic shaping. UnshapedSubnets and ShapedSubnets parameters are mutually exlusive
IngressBurst uint64 `json:"ingressBurst"` // Bandwidth burst in bits for traffic through container. 0 for no limit. If ingressBurst is set, ingressRate must also be set IngressRate uint64 `json:"ingressRate"` // Bandwidth rate in bps for traffic through container. 0 for no limit. If ingressRate is set, ingressBurst must also be set
IngressBurst uint64 `json:"ingressBurst"` // Bandwidth burst in bits for traffic through container. 0 for no limit. If ingressBurst is set, ingressRate must also be set
EgressRate uint64 `json:"egressRate"` // Bandwidth rate in bps for traffic through container. 0 for no limit. If egressRate is set, egressBurst must also be set EgressRate uint64 `json:"egressRate"` // Bandwidth rate in bps for traffic through container. 0 for no limit. If egressRate is set, egressBurst must also be set
EgressBurst uint64 `json:"egressBurst"` // Bandwidth burst in bits for traffic through container. 0 for no limit. If egressBurst is set, egressRate must also be set EgressBurst uint64 `json:"egressBurst"` // Bandwidth burst in bits for traffic through container. 0 for no limit. If egressBurst is set, egressRate must also be set
} }
func (bw *BandwidthEntry) isZero() bool { func (bw *BandwidthEntry) isZero() bool {
@ -103,8 +103,13 @@ func getBandwidth(conf *PluginConf) *BandwidthEntry {
bw = conf.RuntimeConfig.Bandwidth bw = conf.RuntimeConfig.Bandwidth
} }
if bw != nil && bw.NonShapedSubnets == nil { if bw != nil {
bw.NonShapedSubnets = make([]string, 0) if bw.UnshapedSubnets == nil {
bw.UnshapedSubnets = make([]string, 0)
}
if bw.ShapedSubnets == nil {
bw.ShapedSubnets = make([]string, 0)
}
} }
return bw return bw
@ -167,13 +172,25 @@ func getHostInterface(interfaces []*current.Interface, containerIfName string, n
return nil, fmt.Errorf("no veth peer of container interface found in host ns") return nil, fmt.Errorf("no veth peer of container interface found in host ns")
} }
func validateSubnets(subnets []string) error { func validateSubnets(unshapedSubnets []string, shapedSubnets []string) error {
for _, subnet := range subnets { if len(unshapedSubnets) > 0 && len(shapedSubnets) > 0 {
return fmt.Errorf("unshapedSubnets and shapedSubnets cannot be both specified, one of them should be discarded")
}
for _, subnet := range unshapedSubnets {
_, _, err := net.ParseCIDR(subnet) _, _, err := net.ParseCIDR(subnet)
if err != nil { if err != nil {
return fmt.Errorf("bad subnet %q provided, details %s", subnet, err) return fmt.Errorf("bad subnet %q provided, details %s", subnet, err)
} }
} }
for _, subnet := range shapedSubnets {
_, _, err := net.ParseCIDR(subnet)
if err != nil {
return fmt.Errorf("bad subnet %q provided, details %s", subnet, err)
}
}
return nil return nil
} }
@ -188,7 +205,7 @@ func cmdAdd(args *skel.CmdArgs) error {
return types.PrintResult(conf.PrevResult, conf.CNIVersion) return types.PrintResult(conf.PrevResult, conf.CNIVersion)
} }
if err = validateSubnets(bandwidth.NonShapedSubnets); err != nil { if err = validateSubnets(bandwidth.UnshapedSubnets, bandwidth.ShapedSubnets); err != nil {
return err return err
} }
@ -214,7 +231,7 @@ func cmdAdd(args *skel.CmdArgs) error {
if bandwidth.IngressRate > 0 && bandwidth.IngressBurst > 0 { if bandwidth.IngressRate > 0 && bandwidth.IngressBurst > 0 {
err = CreateIngressQdisc(bandwidth.IngressRate, bandwidth.IngressBurst, err = CreateIngressQdisc(bandwidth.IngressRate, bandwidth.IngressBurst,
bandwidth.NonShapedSubnets, hostInterface.Name) bandwidth.UnshapedSubnets, bandwidth.ShapedSubnets, hostInterface.Name)
if err != nil { if err != nil {
return err return err
} }
@ -243,7 +260,8 @@ func cmdAdd(args *skel.CmdArgs) error {
Mac: ifbDevice.Attrs().HardwareAddr.String(), Mac: ifbDevice.Attrs().HardwareAddr.String(),
}) })
err = CreateEgressQdisc(bandwidth.EgressRate, bandwidth.EgressBurst, err = CreateEgressQdisc(bandwidth.EgressRate, bandwidth.EgressBurst,
bandwidth.NonShapedSubnets, hostInterface.Name, ifbDeviceName) bandwidth.UnshapedSubnets, bandwidth.ShapedSubnets, hostInterface.Name,
ifbDeviceName)
if err != nil { if err != nil {
return err return err
} }
@ -316,8 +334,7 @@ func cmdCheck(args *skel.CmdArgs) error {
bandwidth := getBandwidth(bwConf) bandwidth := getBandwidth(bwConf)
err = validateSubnets(bandwidth.NonShapedSubnets) if err = validateSubnets(bandwidth.UnshapedSubnets, bandwidth.ShapedSubnets); err != nil {
if err != nil {
return fmt.Errorf("failed to check subnets, details %s", err) return fmt.Errorf("failed to check subnets, details %s", err)
} }
@ -325,7 +342,7 @@ func cmdCheck(args *skel.CmdArgs) error {
rateInBytes := bandwidth.IngressRate / 8 rateInBytes := bandwidth.IngressRate / 8
burstInBytes := bandwidth.IngressBurst / 8 burstInBytes := bandwidth.IngressBurst / 8
bufferInBytes := buffer(rateInBytes, uint32(burstInBytes)) bufferInBytes := buffer(rateInBytes, uint32(burstInBytes))
err = checkHTB(link, rateInBytes, bufferInBytes) err = checkHTB(link, rateInBytes, bufferInBytes, bandwidth.ShapedSubnets)
if err != nil { if err != nil {
return err return err
} }
@ -339,7 +356,7 @@ func cmdCheck(args *skel.CmdArgs) error {
if err != nil { if err != nil {
return fmt.Errorf("get ifb device: %s", err) return fmt.Errorf("get ifb device: %s", err)
} }
err = checkHTB(ifbDevice, rateInBytes, bufferInBytes) err = checkHTB(ifbDevice, rateInBytes, bufferInBytes, bandwidth.ShapedSubnets)
if err != nil { if err != nil {
return err return err
} }
@ -347,7 +364,7 @@ func cmdCheck(args *skel.CmdArgs) error {
return nil return nil
} }
func checkHTB(link netlink.Link, rateInBytes uint64, bufferInBytes uint32) error { func checkHTB(link netlink.Link, rateInBytes uint64, bufferInBytes uint32, shapedSubnets []string) error {
qdiscs, err := SafeQdiscList(link) qdiscs, err := SafeQdiscList(link)
if err != nil { if err != nil {
return err return err
@ -367,7 +384,12 @@ func checkHTB(link netlink.Link, rateInBytes uint64, bufferInBytes uint32) error
} }
foundHTB = true foundHTB = true
if htb.Defcls != DefaultClassMinorID { defaultClassMinorID := ShapedClassMinorID
if len(shapedSubnets) > 0 {
defaultClassMinorID = UnShapedClassMinorID
}
if htb.Defcls != uint32(defaultClassMinorID) {
return fmt.Errorf("Default class does not match") return fmt.Errorf("Default class does not match")
} }
@ -404,10 +426,7 @@ func checkHTB(link netlink.Link, rateInBytes uint64, bufferInBytes uint32) error
} }
} }
// TODO: check non shaped subnet filters // TODO: check subnet filters
// if bandwidth.NonShapedSubnets {
// filters, err := netlink.FilterList(link, htb.Handle)
// }
} }
return nil return nil