spec/plugins: return interface details and multiple IP addresses to runtime
Updates the spec and plugins to return an array of interfaces and IP details to the runtime including: - interface names and MAC addresses configured by the plugin - whether the interfaces are sandboxed (container/VM) or host (bridge, veth, etc) - multiple IP addresses configured by IPAM and which interface they have been assigned to Returning interface details is useful for runtimes, as well as allowing more flexible chaining of CNI plugins themselves. For example, some meta plugins may need to know the host-side interface to be able to apply firewall or traffic shaping rules to the container.
This commit is contained in:
@ -71,11 +71,12 @@ func (d *DHCP) Allocate(args *skel.CmdArgs, result *current.Result) error {
|
||||
|
||||
d.setLease(args.ContainerID, conf.Name, l)
|
||||
|
||||
result.IP4 = ¤t.IPConfig{
|
||||
IP: *ipn,
|
||||
result.IPs = []*current.IPConfig{{
|
||||
Version: "4",
|
||||
Address: *ipn,
|
||||
Gateway: l.Gateway(),
|
||||
Routes: l.Routes(),
|
||||
}
|
||||
}}
|
||||
result.Routes = l.Routes()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
@ -291,7 +291,7 @@ func (l *DHCPLease) Gateway() net.IP {
|
||||
return parseRouter(l.opts)
|
||||
}
|
||||
|
||||
func (l *DHCPLease) Routes() []types.Route {
|
||||
func (l *DHCPLease) Routes() []*types.Route {
|
||||
routes := parseRoutes(l.opts)
|
||||
return append(routes, parseCIDRRoutes(l.opts)...)
|
||||
}
|
||||
|
@ -21,6 +21,7 @@ import (
|
||||
"path/filepath"
|
||||
|
||||
"github.com/containernetworking/cni/pkg/skel"
|
||||
"github.com/containernetworking/cni/pkg/types"
|
||||
"github.com/containernetworking/cni/pkg/types/current"
|
||||
"github.com/containernetworking/cni/pkg/version"
|
||||
)
|
||||
@ -31,16 +32,24 @@ func main() {
|
||||
if len(os.Args) > 1 && os.Args[1] == "daemon" {
|
||||
runDaemon()
|
||||
} else {
|
||||
skel.PluginMain(cmdAdd, cmdDel, version.Legacy)
|
||||
skel.PluginMain(cmdAdd, cmdDel, version.All)
|
||||
}
|
||||
}
|
||||
|
||||
func cmdAdd(args *skel.CmdArgs) error {
|
||||
// Plugin must return result in same version as specified in netconf
|
||||
versionDecoder := &version.ConfigDecoder{}
|
||||
confVersion, err := versionDecoder.Decode(args.StdinData)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
result := ¤t.Result{}
|
||||
if err := rpcCall("DHCP.Allocate", args, result); err != nil {
|
||||
return err
|
||||
}
|
||||
return result.Print()
|
||||
|
||||
return types.PrintResult(result, confVersion)
|
||||
}
|
||||
|
||||
func cmdDel(args *skel.CmdArgs) error {
|
||||
|
@ -40,17 +40,17 @@ func classfulSubnet(sn net.IP) net.IPNet {
|
||||
}
|
||||
}
|
||||
|
||||
func parseRoutes(opts dhcp4.Options) []types.Route {
|
||||
func parseRoutes(opts dhcp4.Options) []*types.Route {
|
||||
// StaticRoutes format: pairs of:
|
||||
// Dest = 4 bytes; Classful IP subnet
|
||||
// Router = 4 bytes; IP address of router
|
||||
|
||||
routes := []types.Route{}
|
||||
routes := []*types.Route{}
|
||||
if opt, ok := opts[dhcp4.OptionStaticRoute]; ok {
|
||||
for len(opt) >= 8 {
|
||||
sn := opt[0:4]
|
||||
r := opt[4:8]
|
||||
rt := types.Route{
|
||||
rt := &types.Route{
|
||||
Dst: classfulSubnet(sn),
|
||||
GW: r,
|
||||
}
|
||||
@ -62,10 +62,10 @@ func parseRoutes(opts dhcp4.Options) []types.Route {
|
||||
return routes
|
||||
}
|
||||
|
||||
func parseCIDRRoutes(opts dhcp4.Options) []types.Route {
|
||||
func parseCIDRRoutes(opts dhcp4.Options) []*types.Route {
|
||||
// See RFC4332 for format (http://tools.ietf.org/html/rfc3442)
|
||||
|
||||
routes := []types.Route{}
|
||||
routes := []*types.Route{}
|
||||
if opt, ok := opts[dhcp4.OptionClasslessRouteFormat]; ok {
|
||||
for len(opt) >= 5 {
|
||||
width := int(opt[0])
|
||||
@ -89,7 +89,7 @@ func parseCIDRRoutes(opts dhcp4.Options) []types.Route {
|
||||
|
||||
gw := net.IP(opt[octets+1 : octets+5])
|
||||
|
||||
rt := types.Route{
|
||||
rt := &types.Route{
|
||||
Dst: net.IPNet{
|
||||
IP: net.IP(sn),
|
||||
Mask: net.CIDRMask(width, 32),
|
||||
|
@ -22,16 +22,16 @@ import (
|
||||
"github.com/d2g/dhcp4"
|
||||
)
|
||||
|
||||
func validateRoutes(t *testing.T, routes []types.Route) {
|
||||
expected := []types.Route{
|
||||
types.Route{
|
||||
func validateRoutes(t *testing.T, routes []*types.Route) {
|
||||
expected := []*types.Route{
|
||||
&types.Route{
|
||||
Dst: net.IPNet{
|
||||
IP: net.IPv4(10, 0, 0, 0),
|
||||
Mask: net.CIDRMask(8, 32),
|
||||
},
|
||||
GW: net.IPv4(10, 1, 2, 3),
|
||||
},
|
||||
types.Route{
|
||||
&types.Route{
|
||||
Dst: net.IPNet{
|
||||
IP: net.IPv4(192, 168, 1, 0),
|
||||
Mask: net.CIDRMask(24, 32),
|
||||
|
@ -20,6 +20,7 @@ import (
|
||||
"net"
|
||||
|
||||
"github.com/containernetworking/cni/pkg/ip"
|
||||
"github.com/containernetworking/cni/pkg/types"
|
||||
"github.com/containernetworking/cni/pkg/types/current"
|
||||
"github.com/containernetworking/cni/plugins/ipam/host-local/backend"
|
||||
)
|
||||
@ -129,7 +130,7 @@ func validateRangeIP(ip net.IP, ipnet *net.IPNet, start net.IP, end net.IP) erro
|
||||
}
|
||||
|
||||
// Returns newly allocated IP along with its config
|
||||
func (a *IPAllocator) Get(id string) (*current.IPConfig, error) {
|
||||
func (a *IPAllocator) Get(id string) (*current.IPConfig, []*types.Route, error) {
|
||||
a.store.Lock()
|
||||
defer a.store.Unlock()
|
||||
|
||||
@ -145,7 +146,7 @@ func (a *IPAllocator) Get(id string) (*current.IPConfig, error) {
|
||||
|
||||
if requestedIP != nil {
|
||||
if gw != nil && gw.Equal(a.conf.Args.IP) {
|
||||
return nil, fmt.Errorf("requested IP must differ gateway IP")
|
||||
return nil, nil, fmt.Errorf("requested IP must differ gateway IP")
|
||||
}
|
||||
|
||||
subnet := net.IPNet{
|
||||
@ -154,22 +155,24 @@ func (a *IPAllocator) Get(id string) (*current.IPConfig, error) {
|
||||
}
|
||||
err := validateRangeIP(requestedIP, &subnet, a.start, a.end)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
reserved, err := a.store.Reserve(id, requestedIP)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
if reserved {
|
||||
return ¤t.IPConfig{
|
||||
IP: net.IPNet{IP: requestedIP, Mask: a.conf.Subnet.Mask},
|
||||
ipConfig := ¤t.IPConfig{
|
||||
Version: "4",
|
||||
Address: net.IPNet{IP: requestedIP, Mask: a.conf.Subnet.Mask},
|
||||
Gateway: gw,
|
||||
Routes: a.conf.Routes,
|
||||
}, nil
|
||||
}
|
||||
routes := convertRoutesToCurrent(a.conf.Routes)
|
||||
return ipConfig, routes, nil
|
||||
}
|
||||
return nil, fmt.Errorf("requested IP address %q is not available in network: %s", requestedIP, a.conf.Name)
|
||||
return nil, nil, fmt.Errorf("requested IP address %q is not available in network: %s", requestedIP, a.conf.Name)
|
||||
}
|
||||
|
||||
startIP, endIP := a.getSearchRange()
|
||||
@ -181,21 +184,23 @@ func (a *IPAllocator) Get(id string) (*current.IPConfig, error) {
|
||||
|
||||
reserved, err := a.store.Reserve(id, cur)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, nil, err
|
||||
}
|
||||
if reserved {
|
||||
return ¤t.IPConfig{
|
||||
IP: net.IPNet{IP: cur, Mask: a.conf.Subnet.Mask},
|
||||
ipConfig := ¤t.IPConfig{
|
||||
Version: "4",
|
||||
Address: net.IPNet{IP: cur, Mask: a.conf.Subnet.Mask},
|
||||
Gateway: gw,
|
||||
Routes: a.conf.Routes,
|
||||
}, nil
|
||||
}
|
||||
routes := convertRoutesToCurrent(a.conf.Routes)
|
||||
return ipConfig, routes, nil
|
||||
}
|
||||
// break here to complete the loop
|
||||
if cur.Equal(endIP) {
|
||||
break
|
||||
}
|
||||
}
|
||||
return nil, fmt.Errorf("no IP addresses available in network: %s", a.conf.Name)
|
||||
return nil, nil, fmt.Errorf("no IP addresses available in network: %s", a.conf.Name)
|
||||
}
|
||||
|
||||
// Releases all IPs allocated for the container with given ID
|
||||
|
@ -31,10 +31,10 @@ type AllocatorTestCase struct {
|
||||
lastIP string
|
||||
}
|
||||
|
||||
func (t AllocatorTestCase) run() (*current.IPConfig, error) {
|
||||
func (t AllocatorTestCase) run() (*current.IPConfig, []*types.Route, error) {
|
||||
subnet, err := types.ParseCIDR(t.subnet)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
conf := IPAMConfig{
|
||||
@ -45,14 +45,14 @@ func (t AllocatorTestCase) run() (*current.IPConfig, error) {
|
||||
store := fakestore.NewFakeStore(t.ipmap, net.ParseIP(t.lastIP))
|
||||
alloc, err := NewIPAllocator(&conf, store)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, nil, err
|
||||
}
|
||||
res, err := alloc.Get("ID")
|
||||
res, routes, err := alloc.Get("ID")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
return res, nil
|
||||
return res, routes, nil
|
||||
}
|
||||
|
||||
var _ = Describe("host-local ip allocator", func() {
|
||||
@ -129,9 +129,9 @@ var _ = Describe("host-local ip allocator", func() {
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
res, err := tc.run()
|
||||
res, _, err := tc.run()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(res.IP.IP.String()).To(Equal(tc.expectResult))
|
||||
Expect(res.Address.IP.String()).To(Equal(tc.expectResult))
|
||||
}
|
||||
})
|
||||
|
||||
@ -149,14 +149,14 @@ var _ = Describe("host-local ip allocator", func() {
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
for i := 1; i < 254; i++ {
|
||||
res, err := alloc.Get("ID")
|
||||
res, _, err := alloc.Get("ID")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
// i+1 because the gateway address is skipped
|
||||
s := fmt.Sprintf("192.168.1.%d/24", i+1)
|
||||
Expect(s).To(Equal(res.IP.String()))
|
||||
Expect(s).To(Equal(res.Address.String()))
|
||||
}
|
||||
|
||||
_, err = alloc.Get("ID")
|
||||
_, _, err = alloc.Get("ID")
|
||||
Expect(err).To(HaveOccurred())
|
||||
})
|
||||
|
||||
@ -174,13 +174,13 @@ var _ = Describe("host-local ip allocator", func() {
|
||||
alloc, err := NewIPAllocator(&conf, store)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
res, err := alloc.Get("ID")
|
||||
res, _, err := alloc.Get("ID")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(res.IP.String()).To(Equal("192.168.1.10/24"))
|
||||
Expect(res.Address.String()).To(Equal("192.168.1.10/24"))
|
||||
|
||||
res, err = alloc.Get("ID")
|
||||
res, _, err = alloc.Get("ID")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(res.IP.String()).To(Equal("192.168.1.11/24"))
|
||||
Expect(res.Address.String()).To(Equal("192.168.1.11/24"))
|
||||
})
|
||||
|
||||
It("should allocate RangeEnd but not past RangeEnd", func() {
|
||||
@ -198,13 +198,13 @@ var _ = Describe("host-local ip allocator", func() {
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
for i := 1; i < 5; i++ {
|
||||
res, err := alloc.Get("ID")
|
||||
res, _, err := alloc.Get("ID")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
// i+1 because the gateway address is skipped
|
||||
Expect(res.IP.String()).To(Equal(fmt.Sprintf("192.168.1.%d/24", i+1)))
|
||||
Expect(res.Address.String()).To(Equal(fmt.Sprintf("192.168.1.%d/24", i+1)))
|
||||
}
|
||||
|
||||
_, err = alloc.Get("ID")
|
||||
_, _, err = alloc.Get("ID")
|
||||
Expect(err).To(HaveOccurred())
|
||||
})
|
||||
|
||||
@ -222,9 +222,9 @@ var _ = Describe("host-local ip allocator", func() {
|
||||
}
|
||||
store := fakestore.NewFakeStore(ipmap, nil)
|
||||
alloc, _ := NewIPAllocator(&conf, store)
|
||||
res, err := alloc.Get("ID")
|
||||
res, _, err := alloc.Get("ID")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(res.IP.IP.String()).To(Equal(requestedIP.String()))
|
||||
Expect(res.Address.IP.String()).To(Equal(requestedIP.String()))
|
||||
})
|
||||
|
||||
It("must return an error when the requested IP is after RangeEnd", func() {
|
||||
@ -240,7 +240,7 @@ var _ = Describe("host-local ip allocator", func() {
|
||||
}
|
||||
store := fakestore.NewFakeStore(ipmap, nil)
|
||||
alloc, _ := NewIPAllocator(&conf, store)
|
||||
_, err = alloc.Get("ID")
|
||||
_, _, err = alloc.Get("ID")
|
||||
Expect(err).To(HaveOccurred())
|
||||
})
|
||||
|
||||
@ -257,7 +257,7 @@ var _ = Describe("host-local ip allocator", func() {
|
||||
}
|
||||
store := fakestore.NewFakeStore(ipmap, nil)
|
||||
alloc, _ := NewIPAllocator(&conf, store)
|
||||
_, err = alloc.Get("ID")
|
||||
_, _, err = alloc.Get("ID")
|
||||
Expect(err).To(HaveOccurred())
|
||||
})
|
||||
})
|
||||
@ -332,7 +332,7 @@ var _ = Describe("host-local ip allocator", func() {
|
||||
},
|
||||
}
|
||||
for _, tc := range testCases {
|
||||
_, err := tc.run()
|
||||
_, _, err := tc.run()
|
||||
Expect(err).To(MatchError("no IP addresses available in network: test"))
|
||||
}
|
||||
})
|
||||
|
@ -42,31 +42,43 @@ type IPAMArgs struct {
|
||||
}
|
||||
|
||||
type Net struct {
|
||||
Name string `json:"name"`
|
||||
IPAM *IPAMConfig `json:"ipam"`
|
||||
Name string `json:"name"`
|
||||
CNIVersion string `json:"cniVersion"`
|
||||
IPAM *IPAMConfig `json:"ipam"`
|
||||
}
|
||||
|
||||
// NewIPAMConfig creates a NetworkConfig from the given network name.
|
||||
func LoadIPAMConfig(bytes []byte, args string) (*IPAMConfig, error) {
|
||||
func LoadIPAMConfig(bytes []byte, args string) (*IPAMConfig, string, error) {
|
||||
n := Net{}
|
||||
if err := json.Unmarshal(bytes, &n); err != nil {
|
||||
return nil, err
|
||||
return nil, "", err
|
||||
}
|
||||
|
||||
if args != "" {
|
||||
n.IPAM.Args = &IPAMArgs{}
|
||||
err := types.LoadArgs(args, n.IPAM.Args)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, "", err
|
||||
}
|
||||
}
|
||||
|
||||
if n.IPAM == nil {
|
||||
return nil, fmt.Errorf("IPAM config missing 'ipam' key")
|
||||
return nil, "", fmt.Errorf("IPAM config missing 'ipam' key")
|
||||
}
|
||||
|
||||
// Copy net name into IPAM so not to drag Net struct around
|
||||
n.IPAM.Name = n.Name
|
||||
|
||||
return n.IPAM, nil
|
||||
return n.IPAM, n.CNIVersion, nil
|
||||
}
|
||||
|
||||
func convertRoutesToCurrent(routes []types.Route) []*types.Route {
|
||||
var currentRoutes []*types.Route
|
||||
for _, r := range routes {
|
||||
currentRoutes = append(currentRoutes, &types.Route{
|
||||
Dst: r.Dst,
|
||||
GW: r.GW,
|
||||
})
|
||||
}
|
||||
return currentRoutes
|
||||
}
|
||||
|
@ -20,10 +20,12 @@ import (
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/containernetworking/cni/pkg/skel"
|
||||
"github.com/containernetworking/cni/pkg/testutils"
|
||||
"github.com/containernetworking/cni/pkg/types"
|
||||
"github.com/containernetworking/cni/pkg/types/020"
|
||||
"github.com/containernetworking/cni/pkg/types/current"
|
||||
|
||||
. "github.com/onsi/ginkgo"
|
||||
@ -43,7 +45,7 @@ var _ = Describe("host-local Operations", func() {
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
conf := fmt.Sprintf(`{
|
||||
"cniVersion": "0.2.0",
|
||||
"cniVersion": "0.3.0",
|
||||
"name": "mynet",
|
||||
"type": "ipvlan",
|
||||
"master": "foo0",
|
||||
@ -63,19 +65,83 @@ var _ = Describe("host-local Operations", func() {
|
||||
}
|
||||
|
||||
// Allocate the IP
|
||||
r, _, err := testutils.CmdAddWithResult(nspath, ifname, []byte(conf), func() error {
|
||||
r, raw, err := testutils.CmdAddWithResult(nspath, ifname, []byte(conf), func() error {
|
||||
return cmdAdd(args)
|
||||
})
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(strings.Index(string(raw), "\"version\":")).Should(BeNumerically(">", 0))
|
||||
|
||||
result, err := current.GetResult(r)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
expectedAddress, err := types.ParseCIDR("10.1.2.2/24")
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(len(result.IPs)).To(Equal(1))
|
||||
expectedAddress.IP = expectedAddress.IP.To16()
|
||||
Expect(result.IPs[0].Address).To(Equal(*expectedAddress))
|
||||
Expect(result.IPs[0].Gateway).To(Equal(net.ParseIP("10.1.2.1")))
|
||||
|
||||
ipFilePath := filepath.Join(tmpDir, "mynet", "10.1.2.2")
|
||||
contents, err := ioutil.ReadFile(ipFilePath)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(string(contents)).To(Equal("dummy"))
|
||||
|
||||
lastFilePath := filepath.Join(tmpDir, "mynet", "last_reserved_ip")
|
||||
contents, err = ioutil.ReadFile(lastFilePath)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(string(contents)).To(Equal("10.1.2.2"))
|
||||
|
||||
// Release the IP
|
||||
err = testutils.CmdDelWithResult(nspath, ifname, func() error {
|
||||
return cmdDel(args)
|
||||
})
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
_, err = os.Stat(ipFilePath)
|
||||
Expect(err).To(HaveOccurred())
|
||||
})
|
||||
|
||||
It("allocates and releases an address with ADD/DEL and 0.1.0 config", func() {
|
||||
const ifname string = "eth0"
|
||||
const nspath string = "/some/where"
|
||||
|
||||
tmpDir, err := ioutil.TempDir("", "host_local_artifacts")
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
defer os.RemoveAll(tmpDir)
|
||||
|
||||
conf := fmt.Sprintf(`{
|
||||
"cniVersion": "0.1.0",
|
||||
"name": "mynet",
|
||||
"type": "ipvlan",
|
||||
"master": "foo0",
|
||||
"ipam": {
|
||||
"type": "host-local",
|
||||
"subnet": "10.1.2.0/24",
|
||||
"dataDir": "%s"
|
||||
}
|
||||
}`, tmpDir)
|
||||
|
||||
args := &skel.CmdArgs{
|
||||
ContainerID: "dummy",
|
||||
Netns: nspath,
|
||||
IfName: ifname,
|
||||
StdinData: []byte(conf),
|
||||
}
|
||||
|
||||
// Allocate the IP
|
||||
r, raw, err := testutils.CmdAddWithResult(nspath, ifname, []byte(conf), func() error {
|
||||
return cmdAdd(args)
|
||||
})
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(strings.Index(string(raw), "\"ip4\":")).Should(BeNumerically(">", 0))
|
||||
|
||||
result, err := types020.GetResult(r)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
expectedAddress, err := types.ParseCIDR("10.1.2.2/24")
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
expectedAddress.IP = expectedAddress.IP.To16()
|
||||
Expect(result.IP4.IP).To(Equal(*expectedAddress))
|
||||
|
||||
Expect(result.IP4.Gateway).To(Equal(net.ParseIP("10.1.2.1")))
|
||||
|
||||
ipFilePath := filepath.Join(tmpDir, "mynet", "10.1.2.2")
|
||||
@ -136,7 +202,7 @@ var _ = Describe("host-local Operations", func() {
|
||||
result, err := current.GetResult(r)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
ipFilePath := filepath.Join(tmpDir, "mynet", result.IP4.IP.IP.String())
|
||||
ipFilePath := filepath.Join(tmpDir, "mynet", result.IPs[0].Address.IP.String())
|
||||
contents, err := ioutil.ReadFile(ipFilePath)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(string(contents)).To(Equal("dummy"))
|
||||
|
@ -19,28 +19,29 @@ import (
|
||||
"github.com/containernetworking/cni/plugins/ipam/host-local/backend/disk"
|
||||
|
||||
"github.com/containernetworking/cni/pkg/skel"
|
||||
"github.com/containernetworking/cni/pkg/types"
|
||||
"github.com/containernetworking/cni/pkg/types/current"
|
||||
"github.com/containernetworking/cni/pkg/version"
|
||||
)
|
||||
|
||||
func main() {
|
||||
skel.PluginMain(cmdAdd, cmdDel, version.Legacy)
|
||||
skel.PluginMain(cmdAdd, cmdDel, version.All)
|
||||
}
|
||||
|
||||
func cmdAdd(args *skel.CmdArgs) error {
|
||||
ipamConf, err := allocator.LoadIPAMConfig(args.StdinData, args.Args)
|
||||
ipamConf, confVersion, err := allocator.LoadIPAMConfig(args.StdinData, args.Args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
r := ¤t.Result{}
|
||||
result := ¤t.Result{}
|
||||
|
||||
if ipamConf.ResolvConf != "" {
|
||||
dns, err := parseResolvConf(ipamConf.ResolvConf)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
r.DNS = *dns
|
||||
result.DNS = *dns
|
||||
}
|
||||
|
||||
store, err := disk.New(ipamConf.Name, ipamConf.DataDir)
|
||||
@ -54,16 +55,18 @@ func cmdAdd(args *skel.CmdArgs) error {
|
||||
return err
|
||||
}
|
||||
|
||||
r.IP4, err = allocator.Get(args.ContainerID)
|
||||
ipConf, routes, err := allocator.Get(args.ContainerID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
result.IPs = []*current.IPConfig{ipConf}
|
||||
result.Routes = routes
|
||||
|
||||
return r.Print()
|
||||
return types.PrintResult(result, confVersion)
|
||||
}
|
||||
|
||||
func cmdDel(args *skel.CmdArgs) error {
|
||||
ipamConf, err := allocator.LoadIPAMConfig(args.StdinData, args.Args)
|
||||
ipamConf, _, err := allocator.LoadIPAMConfig(args.StdinData, args.Args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
@ -53,14 +53,14 @@ func init() {
|
||||
runtime.LockOSThread()
|
||||
}
|
||||
|
||||
func loadNetConf(bytes []byte) (*NetConf, error) {
|
||||
func loadNetConf(bytes []byte) (*NetConf, string, error) {
|
||||
n := &NetConf{
|
||||
BrName: defaultBrName,
|
||||
}
|
||||
if err := json.Unmarshal(bytes, n); err != nil {
|
||||
return nil, fmt.Errorf("failed to load netconf: %v", err)
|
||||
return nil, "", fmt.Errorf("failed to load netconf: %v", err)
|
||||
}
|
||||
return n, nil
|
||||
return n, n.CNIVersion, nil
|
||||
}
|
||||
|
||||
func ensureBridgeAddr(br *netlink.Bridge, ipn *net.IPNet, forceAddress bool) error {
|
||||
@ -139,16 +139,16 @@ func ensureBridge(brName string, mtu int) (*netlink.Bridge, error) {
|
||||
},
|
||||
}
|
||||
|
||||
if err := netlink.LinkAdd(br); err != nil {
|
||||
if err != syscall.EEXIST {
|
||||
return nil, fmt.Errorf("could not add %q: %v", brName, err)
|
||||
}
|
||||
err := netlink.LinkAdd(br)
|
||||
if err != nil && err != syscall.EEXIST {
|
||||
return nil, fmt.Errorf("could not add %q: %v", brName, err)
|
||||
}
|
||||
|
||||
// it's ok if the device already exists as long as config is similar
|
||||
br, err = bridgeByName(brName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Re-fetch link to read all attributes and if it already existed,
|
||||
// ensure it's really a bridge with similar configuration
|
||||
br, err = bridgeByName(brName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err := netlink.LinkSetUp(br); err != nil {
|
||||
@ -158,40 +158,44 @@ func ensureBridge(brName string, mtu int) (*netlink.Bridge, error) {
|
||||
return br, nil
|
||||
}
|
||||
|
||||
func setupVeth(netns ns.NetNS, br *netlink.Bridge, ifName string, mtu int, hairpinMode bool) error {
|
||||
var hostVethName string
|
||||
func setupVeth(netns ns.NetNS, br *netlink.Bridge, ifName string, mtu int, hairpinMode bool) (*current.Interface, *current.Interface, error) {
|
||||
contIface := ¤t.Interface{}
|
||||
hostIface := ¤t.Interface{}
|
||||
|
||||
err := netns.Do(func(hostNS ns.NetNS) error {
|
||||
// create the veth pair in the container and move host end into host netns
|
||||
hostVeth, _, err := ip.SetupVeth(ifName, mtu, hostNS)
|
||||
hostVeth, containerVeth, err := ip.SetupVeth(ifName, mtu, hostNS)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
hostVethName = hostVeth.Attrs().Name
|
||||
contIface.Name = containerVeth.Attrs().Name
|
||||
contIface.Mac = containerVeth.Attrs().HardwareAddr.String()
|
||||
contIface.Sandbox = netns.Path()
|
||||
hostIface.Name = hostVeth.Attrs().Name
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
// need to lookup hostVeth again as its index has changed during ns move
|
||||
hostVeth, err := netlink.LinkByName(hostVethName)
|
||||
hostVeth, err := netlink.LinkByName(hostIface.Name)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to lookup %q: %v", hostVethName, err)
|
||||
return nil, nil, fmt.Errorf("failed to lookup %q: %v", hostIface.Name, err)
|
||||
}
|
||||
hostIface.Mac = hostVeth.Attrs().HardwareAddr.String()
|
||||
|
||||
// connect host veth end to the bridge
|
||||
if err = netlink.LinkSetMaster(hostVeth, br); err != nil {
|
||||
return fmt.Errorf("failed to connect %q to bridge %v: %v", hostVethName, br.Attrs().Name, err)
|
||||
if err := netlink.LinkSetMaster(hostVeth, br); err != nil {
|
||||
return nil, nil, fmt.Errorf("failed to connect %q to bridge %v: %v", hostVeth.Attrs().Name, br.Attrs().Name, err)
|
||||
}
|
||||
|
||||
// set hairpin mode
|
||||
if err = netlink.LinkSetHairpin(hostVeth, hairpinMode); err != nil {
|
||||
return fmt.Errorf("failed to setup hairpin mode for %v: %v", hostVethName, err)
|
||||
return nil, nil, fmt.Errorf("failed to setup hairpin mode for %v: %v", hostVeth.Attrs().Name, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
return hostIface, contIface, nil
|
||||
}
|
||||
|
||||
func calcGatewayIP(ipn *net.IPNet) net.IP {
|
||||
@ -199,18 +203,21 @@ func calcGatewayIP(ipn *net.IPNet) net.IP {
|
||||
return ip.NextIP(nid)
|
||||
}
|
||||
|
||||
func setupBridge(n *NetConf) (*netlink.Bridge, error) {
|
||||
func setupBridge(n *NetConf) (*netlink.Bridge, *current.Interface, error) {
|
||||
// create bridge if necessary
|
||||
br, err := ensureBridge(n.BrName, n.MTU)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create bridge %q: %v", n.BrName, err)
|
||||
return nil, nil, fmt.Errorf("failed to create bridge %q: %v", n.BrName, err)
|
||||
}
|
||||
|
||||
return br, nil
|
||||
return br, ¤t.Interface{
|
||||
Name: br.Attrs().Name,
|
||||
Mac: br.Attrs().HardwareAddr.String(),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func cmdAdd(args *skel.CmdArgs) error {
|
||||
n, err := loadNetConf(args.StdinData)
|
||||
n, cniVersion, err := loadNetConf(args.StdinData)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@ -219,7 +226,7 @@ func cmdAdd(args *skel.CmdArgs) error {
|
||||
n.IsGW = true
|
||||
}
|
||||
|
||||
br, err := setupBridge(n)
|
||||
br, brInterface, err := setupBridge(n)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@ -230,7 +237,8 @@ func cmdAdd(args *skel.CmdArgs) error {
|
||||
}
|
||||
defer netns.Close()
|
||||
|
||||
if err = setupVeth(netns, br, args.IfName, n.MTU, n.HairpinMode); err != nil {
|
||||
hostInterface, containerInterface, err := setupVeth(netns, br, args.IfName, n.MTU, n.HairpinMode)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@ -240,72 +248,100 @@ func cmdAdd(args *skel.CmdArgs) error {
|
||||
return err
|
||||
}
|
||||
|
||||
result, err := current.GetResult(r)
|
||||
// Convert whatever the IPAM result was into the current Result type
|
||||
result, err := current.NewResultFromResult(r)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// TODO: make this optional when IPv6 is supported
|
||||
if result.IP4 == nil {
|
||||
return errors.New("IPAM plugin returned missing IPv4 config")
|
||||
if len(result.IPs) == 0 {
|
||||
return errors.New("IPAM plugin returned missing IP config")
|
||||
}
|
||||
|
||||
if result.IP4.Gateway == nil && n.IsGW {
|
||||
result.IP4.Gateway = calcGatewayIP(&result.IP4.IP)
|
||||
result.Interfaces = []*current.Interface{brInterface, hostInterface, containerInterface}
|
||||
|
||||
for _, ipc := range result.IPs {
|
||||
// All IPs currently refer to the container interface
|
||||
ipc.Interface = 2
|
||||
if ipc.Gateway == nil && n.IsGW {
|
||||
ipc.Gateway = calcGatewayIP(&ipc.Address)
|
||||
}
|
||||
}
|
||||
|
||||
if err := netns.Do(func(_ ns.NetNS) error {
|
||||
// set the default gateway if requested
|
||||
if n.IsDefaultGW {
|
||||
_, defaultNet, err := net.ParseCIDR("0.0.0.0/0")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, ipc := range result.IPs {
|
||||
defaultNet := &net.IPNet{}
|
||||
switch {
|
||||
case ipc.Address.IP.To4() != nil:
|
||||
defaultNet.IP = net.IPv4zero
|
||||
defaultNet.Mask = net.IPMask(net.IPv4zero)
|
||||
case len(ipc.Address.IP) == net.IPv6len && ipc.Address.IP.To4() == nil:
|
||||
defaultNet.IP = net.IPv6zero
|
||||
defaultNet.Mask = net.IPMask(net.IPv6zero)
|
||||
default:
|
||||
return fmt.Errorf("Unknown IP object: %v", ipc)
|
||||
}
|
||||
|
||||
for _, route := range result.IP4.Routes {
|
||||
if defaultNet.String() == route.Dst.String() {
|
||||
if route.GW != nil && !route.GW.Equal(result.IP4.Gateway) {
|
||||
return fmt.Errorf(
|
||||
"isDefaultGateway ineffective because IPAM sets default route via %q",
|
||||
route.GW,
|
||||
)
|
||||
for _, route := range result.Routes {
|
||||
if defaultNet.String() == route.Dst.String() {
|
||||
if route.GW != nil && !route.GW.Equal(ipc.Gateway) {
|
||||
return fmt.Errorf(
|
||||
"isDefaultGateway ineffective because IPAM sets default route via %q",
|
||||
route.GW,
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
result.Routes = append(
|
||||
result.Routes,
|
||||
&types.Route{Dst: *defaultNet, GW: ipc.Gateway},
|
||||
)
|
||||
}
|
||||
|
||||
result.IP4.Routes = append(
|
||||
result.IP4.Routes,
|
||||
types.Route{Dst: *defaultNet, GW: result.IP4.Gateway},
|
||||
)
|
||||
|
||||
// TODO: IPV6
|
||||
}
|
||||
|
||||
if err := ipam.ConfigureIface(args.IfName, result); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := ip.SetHWAddrByIP(args.IfName, result.IP4.IP.IP, nil /* TODO IPv6 */); err != nil {
|
||||
if err := ip.SetHWAddrByIP(args.IfName, result.IPs[0].Address.IP, nil /* TODO IPv6 */); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Refetch the veth since its MAC address may changed
|
||||
link, err := netlink.LinkByName(args.IfName)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not lookup %q: %v", args.IfName, err)
|
||||
}
|
||||
containerInterface.Mac = link.Attrs().HardwareAddr.String()
|
||||
|
||||
return nil
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if n.IsGW {
|
||||
gwn := &net.IPNet{
|
||||
IP: result.IP4.Gateway,
|
||||
Mask: result.IP4.IP.Mask,
|
||||
var firstV4Addr net.IP
|
||||
for _, ipc := range result.IPs {
|
||||
gwn := &net.IPNet{
|
||||
IP: ipc.Gateway,
|
||||
Mask: ipc.Address.Mask,
|
||||
}
|
||||
if ipc.Gateway.To4() != nil && firstV4Addr == nil {
|
||||
firstV4Addr = ipc.Gateway
|
||||
}
|
||||
|
||||
if err = ensureBridgeAddr(br, gwn, n.ForceAddress); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if err = ensureBridgeAddr(br, gwn, n.ForceAddress); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := ip.SetHWAddrByIP(n.BrName, gwn.IP, nil /* TODO IPv6 */); err != nil {
|
||||
return err
|
||||
if firstV4Addr != nil {
|
||||
if err := ip.SetHWAddrByIP(n.BrName, firstV4Addr, nil /* TODO IPv6 */); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if err := ip.EnableIP4Forward(); err != nil {
|
||||
@ -316,17 +352,28 @@ func cmdAdd(args *skel.CmdArgs) error {
|
||||
if n.IPMasq {
|
||||
chain := utils.FormatChainName(n.Name, args.ContainerID)
|
||||
comment := utils.FormatComment(n.Name, args.ContainerID)
|
||||
if err = ip.SetupIPMasq(ip.Network(&result.IP4.IP), chain, comment); err != nil {
|
||||
return err
|
||||
for _, ipc := range result.IPs {
|
||||
if err = ip.SetupIPMasq(ip.Network(&ipc.Address), chain, comment); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Refetch the bridge since its MAC address may change when the first
|
||||
// veth is added or after its IP address is set
|
||||
br, err = bridgeByName(n.BrName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
brInterface.Mac = br.Attrs().HardwareAddr.String()
|
||||
|
||||
result.DNS = n.DNS
|
||||
return result.Print()
|
||||
|
||||
return types.PrintResult(result, cniVersion)
|
||||
}
|
||||
|
||||
func cmdDel(args *skel.CmdArgs) error {
|
||||
n, err := loadNetConf(args.StdinData)
|
||||
n, _, err := loadNetConf(args.StdinData)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@ -361,5 +408,5 @@ func cmdDel(args *skel.CmdArgs) error {
|
||||
}
|
||||
|
||||
func main() {
|
||||
skel.PluginMain(cmdAdd, cmdDel, version.Legacy)
|
||||
skel.PluginMain(cmdAdd, cmdDel, version.All)
|
||||
}
|
||||
|
@ -17,12 +17,15 @@ package main
|
||||
import (
|
||||
"fmt"
|
||||
"net"
|
||||
"strings"
|
||||
"syscall"
|
||||
|
||||
"github.com/containernetworking/cni/pkg/ns"
|
||||
"github.com/containernetworking/cni/pkg/skel"
|
||||
"github.com/containernetworking/cni/pkg/testutils"
|
||||
"github.com/containernetworking/cni/pkg/types"
|
||||
"github.com/containernetworking/cni/pkg/types/020"
|
||||
"github.com/containernetworking/cni/pkg/types/current"
|
||||
|
||||
"github.com/containernetworking/cni/pkg/utils/hwaddr"
|
||||
|
||||
@ -51,7 +54,7 @@ var _ = Describe("bridge Operations", func() {
|
||||
|
||||
conf := &NetConf{
|
||||
NetConf: types.NetConf{
|
||||
CNIVersion: "0.2.0",
|
||||
CNIVersion: "0.3.0",
|
||||
Name: "testConfig",
|
||||
Type: "bridge",
|
||||
},
|
||||
@ -64,7 +67,7 @@ var _ = Describe("bridge Operations", func() {
|
||||
err := originalNS.Do(func(ns.NetNS) error {
|
||||
defer GinkgoRecover()
|
||||
|
||||
bridge, err := setupBridge(conf)
|
||||
bridge, _, err := setupBridge(conf)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(bridge.Attrs().Name).To(Equal(IFNAME))
|
||||
|
||||
@ -96,7 +99,7 @@ var _ = Describe("bridge Operations", func() {
|
||||
|
||||
conf := &NetConf{
|
||||
NetConf: types.NetConf{
|
||||
CNIVersion: "0.2.0",
|
||||
CNIVersion: "0.3.0",
|
||||
Name: "testConfig",
|
||||
Type: "bridge",
|
||||
},
|
||||
@ -105,7 +108,7 @@ var _ = Describe("bridge Operations", func() {
|
||||
IPMasq: false,
|
||||
}
|
||||
|
||||
bridge, err := setupBridge(conf)
|
||||
bridge, _, err := setupBridge(conf)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(bridge.Attrs().Name).To(Equal(IFNAME))
|
||||
Expect(bridge.Attrs().Index).To(Equal(ifindex))
|
||||
@ -128,7 +131,7 @@ var _ = Describe("bridge Operations", func() {
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
conf := fmt.Sprintf(`{
|
||||
"cniVersion": "0.2.0",
|
||||
"cniVersion": "0.3.0",
|
||||
"name": "mynet",
|
||||
"type": "bridge",
|
||||
"bridge": "%s",
|
||||
@ -151,18 +154,31 @@ var _ = Describe("bridge Operations", func() {
|
||||
StdinData: []byte(conf),
|
||||
}
|
||||
|
||||
var result *current.Result
|
||||
err = originalNS.Do(func(ns.NetNS) error {
|
||||
defer GinkgoRecover()
|
||||
|
||||
_, _, err := testutils.CmdAddWithResult(targetNs.Path(), IFNAME, []byte(conf), func() error {
|
||||
r, raw, err := testutils.CmdAddWithResult(targetNs.Path(), IFNAME, []byte(conf), func() error {
|
||||
return cmdAdd(args)
|
||||
})
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(strings.Index(string(raw), "\"interfaces\":")).Should(BeNumerically(">", 0))
|
||||
|
||||
result, err = current.GetResult(r)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
Expect(len(result.Interfaces)).To(Equal(3))
|
||||
Expect(result.Interfaces[0].Name).To(Equal(BRNAME))
|
||||
Expect(result.Interfaces[2].Name).To(Equal(IFNAME))
|
||||
|
||||
// Make sure bridge link exists
|
||||
link, err := netlink.LinkByName(BRNAME)
|
||||
link, err := netlink.LinkByName(result.Interfaces[0].Name)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(link.Attrs().Name).To(Equal(BRNAME))
|
||||
Expect(link).To(BeAssignableToTypeOf(&netlink.Bridge{}))
|
||||
Expect(link.Attrs().HardwareAddr.String()).To(Equal(result.Interfaces[0].Mac))
|
||||
hwAddr := fmt.Sprintf("%s", link.Attrs().HardwareAddr)
|
||||
Expect(hwAddr).To(HavePrefix(hwaddr.PrivateMACPrefixString))
|
||||
|
||||
// Ensure bridge has gateway address
|
||||
addrs, err := netlink.AddrList(link, syscall.AF_INET)
|
||||
@ -183,23 +199,10 @@ var _ = Describe("bridge Operations", func() {
|
||||
links, err := netlink.LinkList()
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(len(links)).To(Equal(3)) // Bridge, veth, and loopback
|
||||
for _, l := range links {
|
||||
switch {
|
||||
case l.Attrs().Name == BRNAME:
|
||||
{
|
||||
_, isBridge := l.(*netlink.Bridge)
|
||||
Expect(isBridge).To(Equal(true))
|
||||
hwAddr := fmt.Sprintf("%s", l.Attrs().HardwareAddr)
|
||||
Expect(hwAddr).To(HavePrefix(hwaddr.PrivateMACPrefixString))
|
||||
}
|
||||
case l.Attrs().Name != BRNAME && l.Attrs().Name != "lo":
|
||||
{
|
||||
_, isVeth := l.(*netlink.Veth)
|
||||
Expect(isVeth).To(Equal(true))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
link, err = netlink.LinkByName(result.Interfaces[1].Name)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(link).To(BeAssignableToTypeOf(&netlink.Veth{}))
|
||||
return nil
|
||||
})
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
@ -211,6 +214,11 @@ var _ = Describe("bridge Operations", func() {
|
||||
link, err := netlink.LinkByName(IFNAME)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(link.Attrs().Name).To(Equal(IFNAME))
|
||||
Expect(link).To(BeAssignableToTypeOf(&netlink.Veth{}))
|
||||
|
||||
addrs, err := netlink.AddrList(link, syscall.AF_INET)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(len(addrs)).To(Equal(1))
|
||||
|
||||
hwAddr := fmt.Sprintf("%s", link.Attrs().HardwareAddr)
|
||||
Expect(hwAddr).To(HavePrefix(hwaddr.PrivateMACPrefixString))
|
||||
@ -243,7 +251,7 @@ var _ = Describe("bridge Operations", func() {
|
||||
})
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
// Make sure macvlan link has been deleted
|
||||
// Make sure the host veth has been deleted
|
||||
err = targetNs.Do(func(ns.NetNS) error {
|
||||
defer GinkgoRecover()
|
||||
|
||||
@ -253,6 +261,150 @@ var _ = Describe("bridge Operations", func() {
|
||||
return nil
|
||||
})
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
// Make sure the container veth has been deleted
|
||||
err = originalNS.Do(func(ns.NetNS) error {
|
||||
defer GinkgoRecover()
|
||||
|
||||
link, err := netlink.LinkByName(result.Interfaces[1].Name)
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(link).To(BeNil())
|
||||
return nil
|
||||
})
|
||||
})
|
||||
|
||||
It("configures and deconfigures a bridge and veth with default route with ADD/DEL for 0.1.0 config", func() {
|
||||
const BRNAME = "cni0"
|
||||
const IFNAME = "eth0"
|
||||
|
||||
gwaddr, subnet, err := net.ParseCIDR("10.1.2.1/24")
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
conf := fmt.Sprintf(`{
|
||||
"cniVersion": "0.1.0",
|
||||
"name": "mynet",
|
||||
"type": "bridge",
|
||||
"bridge": "%s",
|
||||
"isDefaultGateway": true,
|
||||
"ipMasq": false,
|
||||
"ipam": {
|
||||
"type": "host-local",
|
||||
"subnet": "%s"
|
||||
}
|
||||
}`, BRNAME, subnet.String())
|
||||
|
||||
targetNs, err := ns.NewNS()
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
defer targetNs.Close()
|
||||
|
||||
args := &skel.CmdArgs{
|
||||
ContainerID: "dummy",
|
||||
Netns: targetNs.Path(),
|
||||
IfName: IFNAME,
|
||||
StdinData: []byte(conf),
|
||||
}
|
||||
|
||||
var result *types020.Result
|
||||
err = originalNS.Do(func(ns.NetNS) error {
|
||||
defer GinkgoRecover()
|
||||
|
||||
r, raw, err := testutils.CmdAddWithResult(targetNs.Path(), IFNAME, []byte(conf), func() error {
|
||||
return cmdAdd(args)
|
||||
})
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(strings.Index(string(raw), "\"ip4\":")).Should(BeNumerically(">", 0))
|
||||
|
||||
// We expect a version 0.1.0 result
|
||||
result, err = types020.GetResult(r)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
// Make sure bridge link exists
|
||||
link, err := netlink.LinkByName(BRNAME)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(link.Attrs().Name).To(Equal(BRNAME))
|
||||
Expect(link).To(BeAssignableToTypeOf(&netlink.Bridge{}))
|
||||
hwAddr := fmt.Sprintf("%s", link.Attrs().HardwareAddr)
|
||||
Expect(hwAddr).To(HavePrefix(hwaddr.PrivateMACPrefixString))
|
||||
|
||||
// Ensure bridge has gateway address
|
||||
addrs, err := netlink.AddrList(link, syscall.AF_INET)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(len(addrs)).To(BeNumerically(">", 0))
|
||||
found := false
|
||||
subnetPrefix, subnetBits := subnet.Mask.Size()
|
||||
for _, a := range addrs {
|
||||
aPrefix, aBits := a.IPNet.Mask.Size()
|
||||
if a.IPNet.IP.Equal(gwaddr) && aPrefix == subnetPrefix && aBits == subnetBits {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
Expect(found).To(Equal(true))
|
||||
|
||||
// Check for the veth link in the main namespace; can't
|
||||
// check the for the specific link since version 0.1.0
|
||||
// doesn't report interfaces
|
||||
links, err := netlink.LinkList()
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(len(links)).To(Equal(3)) // Bridge, veth, and loopback
|
||||
return nil
|
||||
})
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
// Find the veth peer in the container namespace and the default route
|
||||
err = targetNs.Do(func(ns.NetNS) error {
|
||||
defer GinkgoRecover()
|
||||
|
||||
link, err := netlink.LinkByName(IFNAME)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(link.Attrs().Name).To(Equal(IFNAME))
|
||||
Expect(link).To(BeAssignableToTypeOf(&netlink.Veth{}))
|
||||
|
||||
addrs, err := netlink.AddrList(link, syscall.AF_INET)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(len(addrs)).To(Equal(1))
|
||||
|
||||
hwAddr := fmt.Sprintf("%s", link.Attrs().HardwareAddr)
|
||||
Expect(hwAddr).To(HavePrefix(hwaddr.PrivateMACPrefixString))
|
||||
|
||||
// Ensure the default route
|
||||
routes, err := netlink.RouteList(link, 0)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
var defaultRouteFound bool
|
||||
for _, route := range routes {
|
||||
defaultRouteFound = (route.Dst == nil && route.Src == nil && route.Gw.Equal(gwaddr))
|
||||
if defaultRouteFound {
|
||||
break
|
||||
}
|
||||
}
|
||||
Expect(defaultRouteFound).To(Equal(true))
|
||||
|
||||
return nil
|
||||
})
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
err = originalNS.Do(func(ns.NetNS) error {
|
||||
defer GinkgoRecover()
|
||||
|
||||
err := testutils.CmdDelWithResult(targetNs.Path(), IFNAME, func() error {
|
||||
return cmdDel(args)
|
||||
})
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
return nil
|
||||
})
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
// Make sure the container veth has been deleted; cannot check
|
||||
// host veth as version 0.1.0 can't report its name
|
||||
err = originalNS.Do(func(ns.NetNS) error {
|
||||
defer GinkgoRecover()
|
||||
|
||||
link, err := netlink.LinkByName(IFNAME)
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(link).To(BeNil())
|
||||
return nil
|
||||
})
|
||||
})
|
||||
|
||||
It("ensure bridge address", func() {
|
||||
@ -262,7 +414,7 @@ var _ = Describe("bridge Operations", func() {
|
||||
|
||||
conf := &NetConf{
|
||||
NetConf: types.NetConf{
|
||||
CNIVersion: "0.2.0",
|
||||
CNIVersion: "0.3.0",
|
||||
Name: "testConfig",
|
||||
Type: "bridge",
|
||||
},
|
||||
@ -285,7 +437,7 @@ var _ = Describe("bridge Operations", func() {
|
||||
err := originalNS.Do(func(ns.NetNS) error {
|
||||
defer GinkgoRecover()
|
||||
|
||||
bridge, err := setupBridge(conf)
|
||||
bridge, _, err := setupBridge(conf)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
// Check if ForceAddress has default value
|
||||
Expect(conf.ForceAddress).To(Equal(false))
|
||||
|
@ -44,15 +44,15 @@ func init() {
|
||||
runtime.LockOSThread()
|
||||
}
|
||||
|
||||
func loadConf(bytes []byte) (*NetConf, error) {
|
||||
func loadConf(bytes []byte) (*NetConf, string, error) {
|
||||
n := &NetConf{}
|
||||
if err := json.Unmarshal(bytes, n); err != nil {
|
||||
return nil, fmt.Errorf("failed to load netconf: %v", err)
|
||||
return nil, "", fmt.Errorf("failed to load netconf: %v", err)
|
||||
}
|
||||
if n.Master == "" {
|
||||
return nil, fmt.Errorf(`"master" field is required. It specifies the host interface name to virtualize`)
|
||||
return nil, "", fmt.Errorf(`"master" field is required. It specifies the host interface name to virtualize`)
|
||||
}
|
||||
return n, nil
|
||||
return n, n.CNIVersion, nil
|
||||
}
|
||||
|
||||
func modeFromString(s string) (netlink.IPVlanMode, error) {
|
||||
@ -68,22 +68,24 @@ func modeFromString(s string) (netlink.IPVlanMode, error) {
|
||||
}
|
||||
}
|
||||
|
||||
func createIpvlan(conf *NetConf, ifName string, netns ns.NetNS) error {
|
||||
func createIpvlan(conf *NetConf, ifName string, netns ns.NetNS) (*current.Interface, error) {
|
||||
ipvlan := ¤t.Interface{}
|
||||
|
||||
mode, err := modeFromString(conf.Mode)
|
||||
if err != nil {
|
||||
return err
|
||||
return nil, err
|
||||
}
|
||||
|
||||
m, err := netlink.LinkByName(conf.Master)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to lookup master %q: %v", conf.Master, err)
|
||||
return nil, fmt.Errorf("failed to lookup master %q: %v", conf.Master, err)
|
||||
}
|
||||
|
||||
// due to kernel bug we have to create with tmpname or it might
|
||||
// collide with the name on the host and error out
|
||||
tmpName, err := ip.RandomVethName()
|
||||
if err != nil {
|
||||
return err
|
||||
return nil, err
|
||||
}
|
||||
|
||||
mv := &netlink.IPVlan{
|
||||
@ -97,20 +99,35 @@ func createIpvlan(conf *NetConf, ifName string, netns ns.NetNS) error {
|
||||
}
|
||||
|
||||
if err := netlink.LinkAdd(mv); err != nil {
|
||||
return fmt.Errorf("failed to create ipvlan: %v", err)
|
||||
return nil, fmt.Errorf("failed to create ipvlan: %v", err)
|
||||
}
|
||||
|
||||
return netns.Do(func(_ ns.NetNS) error {
|
||||
err = netns.Do(func(_ ns.NetNS) error {
|
||||
err := ip.RenameLink(tmpName, ifName)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to rename ipvlan to %q: %v", ifName, err)
|
||||
}
|
||||
ipvlan.Name = ifName
|
||||
|
||||
// Re-fetch ipvlan to get all properties/attributes
|
||||
contIpvlan, err := netlink.LinkByName(ipvlan.Name)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to refetch ipvlan %q: %v", ipvlan.Name, err)
|
||||
}
|
||||
ipvlan.Mac = contIpvlan.Attrs().HardwareAddr.String()
|
||||
ipvlan.Sandbox = netns.Path()
|
||||
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return ipvlan, nil
|
||||
}
|
||||
|
||||
func cmdAdd(args *skel.CmdArgs) error {
|
||||
n, err := loadConf(args.StdinData)
|
||||
n, cniVersion, err := loadConf(args.StdinData)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@ -121,7 +138,8 @@ func cmdAdd(args *skel.CmdArgs) error {
|
||||
}
|
||||
defer netns.Close()
|
||||
|
||||
if err = createIpvlan(n, args.IfName, netns); err != nil {
|
||||
ipvlanInterface, err := createIpvlan(n, args.IfName, netns)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@ -130,14 +148,21 @@ func cmdAdd(args *skel.CmdArgs) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
result, err := current.GetResult(r)
|
||||
// Convert whatever the IPAM result was into the current Result type
|
||||
result, err := current.NewResultFromResult(r)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if result.IP4 == nil {
|
||||
return errors.New("IPAM plugin returned missing IPv4 config")
|
||||
if len(result.IPs) == 0 {
|
||||
return errors.New("IPAM plugin returned missing IP config")
|
||||
}
|
||||
for _, ipc := range result.IPs {
|
||||
// All addresses belong to the ipvlan interface
|
||||
ipc.Interface = 0
|
||||
}
|
||||
|
||||
result.Interfaces = []*current.Interface{ipvlanInterface}
|
||||
|
||||
err = netns.Do(func(_ ns.NetNS) error {
|
||||
return ipam.ConfigureIface(args.IfName, result)
|
||||
@ -147,11 +172,12 @@ func cmdAdd(args *skel.CmdArgs) error {
|
||||
}
|
||||
|
||||
result.DNS = n.DNS
|
||||
return result.Print()
|
||||
|
||||
return types.PrintResult(result, cniVersion)
|
||||
}
|
||||
|
||||
func cmdDel(args *skel.CmdArgs) error {
|
||||
n, err := loadConf(args.StdinData)
|
||||
n, _, err := loadConf(args.StdinData)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@ -171,5 +197,5 @@ func cmdDel(args *skel.CmdArgs) error {
|
||||
}
|
||||
|
||||
func main() {
|
||||
skel.PluginMain(cmdAdd, cmdDel, version.Legacy)
|
||||
skel.PluginMain(cmdAdd, cmdDel, version.All)
|
||||
}
|
||||
|
@ -16,11 +16,14 @@ package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net"
|
||||
"syscall"
|
||||
|
||||
"github.com/containernetworking/cni/pkg/ns"
|
||||
"github.com/containernetworking/cni/pkg/skel"
|
||||
"github.com/containernetworking/cni/pkg/testutils"
|
||||
"github.com/containernetworking/cni/pkg/types"
|
||||
"github.com/containernetworking/cni/pkg/types/current"
|
||||
|
||||
"github.com/vishvananda/netlink"
|
||||
|
||||
@ -63,7 +66,7 @@ var _ = Describe("ipvlan Operations", func() {
|
||||
It("creates an ipvlan link in a non-default namespace", func() {
|
||||
conf := &NetConf{
|
||||
NetConf: types.NetConf{
|
||||
CNIVersion: "0.2.0",
|
||||
CNIVersion: "0.3.0",
|
||||
Name: "testConfig",
|
||||
Type: "ipvlan",
|
||||
},
|
||||
@ -80,10 +83,11 @@ var _ = Describe("ipvlan Operations", func() {
|
||||
err = originalNS.Do(func(ns.NetNS) error {
|
||||
defer GinkgoRecover()
|
||||
|
||||
err := createIpvlan(conf, "foobar0", targetNs)
|
||||
_, err := createIpvlan(conf, "foobar0", targetNs)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
return nil
|
||||
})
|
||||
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
// Make sure ipvlan link exists in the target namespace
|
||||
@ -102,7 +106,7 @@ var _ = Describe("ipvlan Operations", func() {
|
||||
const IFNAME = "ipvl0"
|
||||
|
||||
conf := fmt.Sprintf(`{
|
||||
"cniVersion": "0.2.0",
|
||||
"cniVersion": "0.3.0",
|
||||
"name": "mynet",
|
||||
"type": "ipvlan",
|
||||
"master": "%s",
|
||||
@ -123,13 +127,21 @@ var _ = Describe("ipvlan Operations", func() {
|
||||
StdinData: []byte(conf),
|
||||
}
|
||||
|
||||
var result *current.Result
|
||||
err = originalNS.Do(func(ns.NetNS) error {
|
||||
defer GinkgoRecover()
|
||||
|
||||
_, _, err := testutils.CmdAddWithResult(targetNs.Path(), IFNAME, []byte(conf), func() error {
|
||||
r, _, err := testutils.CmdAddWithResult(targetNs.Path(), IFNAME, []byte(conf), func() error {
|
||||
return cmdAdd(args)
|
||||
})
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
result, err = current.GetResult(r)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
Expect(len(result.Interfaces)).To(Equal(1))
|
||||
Expect(result.Interfaces[0].Name).To(Equal(IFNAME))
|
||||
Expect(len(result.IPs)).To(Equal(1))
|
||||
return nil
|
||||
})
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
@ -141,6 +153,14 @@ var _ = Describe("ipvlan Operations", func() {
|
||||
link, err := netlink.LinkByName(IFNAME)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(link.Attrs().Name).To(Equal(IFNAME))
|
||||
|
||||
hwaddr, err := net.ParseMAC(result.Interfaces[0].Mac)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(link.Attrs().HardwareAddr).To(Equal(hwaddr))
|
||||
|
||||
addrs, err := netlink.AddrList(link, syscall.AF_INET)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(len(addrs)).To(Equal(1))
|
||||
return nil
|
||||
})
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
@ -68,5 +68,5 @@ func cmdDel(args *skel.CmdArgs) error {
|
||||
}
|
||||
|
||||
func main() {
|
||||
skel.PluginMain(cmdAdd, cmdDel, version.Legacy)
|
||||
skel.PluginMain(cmdAdd, cmdDel, version.All)
|
||||
}
|
||||
|
@ -18,6 +18,7 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"runtime"
|
||||
|
||||
"github.com/containernetworking/cni/pkg/ip"
|
||||
@ -49,15 +50,15 @@ func init() {
|
||||
runtime.LockOSThread()
|
||||
}
|
||||
|
||||
func loadConf(bytes []byte) (*NetConf, error) {
|
||||
func loadConf(bytes []byte) (*NetConf, string, error) {
|
||||
n := &NetConf{}
|
||||
if err := json.Unmarshal(bytes, n); err != nil {
|
||||
return nil, fmt.Errorf("failed to load netconf: %v", err)
|
||||
return nil, "", fmt.Errorf("failed to load netconf: %v", err)
|
||||
}
|
||||
if n.Master == "" {
|
||||
return nil, fmt.Errorf(`"master" field is required. It specifies the host interface name to virtualize`)
|
||||
return nil, "", fmt.Errorf(`"master" field is required. It specifies the host interface name to virtualize`)
|
||||
}
|
||||
return n, nil
|
||||
return n, n.CNIVersion, nil
|
||||
}
|
||||
|
||||
func modeFromString(s string) (netlink.MacvlanMode, error) {
|
||||
@ -75,22 +76,24 @@ func modeFromString(s string) (netlink.MacvlanMode, error) {
|
||||
}
|
||||
}
|
||||
|
||||
func createMacvlan(conf *NetConf, ifName string, netns ns.NetNS) error {
|
||||
func createMacvlan(conf *NetConf, ifName string, netns ns.NetNS) (*current.Interface, error) {
|
||||
macvlan := ¤t.Interface{}
|
||||
|
||||
mode, err := modeFromString(conf.Mode)
|
||||
if err != nil {
|
||||
return err
|
||||
return nil, err
|
||||
}
|
||||
|
||||
m, err := netlink.LinkByName(conf.Master)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to lookup master %q: %v", conf.Master, err)
|
||||
return nil, fmt.Errorf("failed to lookup master %q: %v", conf.Master, err)
|
||||
}
|
||||
|
||||
// due to kernel bug we have to create with tmpName or it might
|
||||
// collide with the name on the host and error out
|
||||
tmpName, err := ip.RandomVethName()
|
||||
if err != nil {
|
||||
return err
|
||||
return nil, err
|
||||
}
|
||||
|
||||
mv := &netlink.Macvlan{
|
||||
@ -104,10 +107,10 @@ func createMacvlan(conf *NetConf, ifName string, netns ns.NetNS) error {
|
||||
}
|
||||
|
||||
if err := netlink.LinkAdd(mv); err != nil {
|
||||
return fmt.Errorf("failed to create macvlan: %v", err)
|
||||
return nil, fmt.Errorf("failed to create macvlan: %v", err)
|
||||
}
|
||||
|
||||
return netns.Do(func(_ ns.NetNS) error {
|
||||
err = netns.Do(func(_ ns.NetNS) error {
|
||||
// TODO: duplicate following lines for ipv6 support, when it will be added in other places
|
||||
ipv4SysctlValueName := fmt.Sprintf(IPv4InterfaceArpProxySysctlTemplate, tmpName)
|
||||
if _, err := sysctl.Sysctl(ipv4SysctlValueName, "1"); err != nil {
|
||||
@ -121,12 +124,27 @@ func createMacvlan(conf *NetConf, ifName string, netns ns.NetNS) error {
|
||||
_ = netlink.LinkDel(mv)
|
||||
return fmt.Errorf("failed to rename macvlan to %q: %v", ifName, err)
|
||||
}
|
||||
macvlan.Name = ifName
|
||||
|
||||
// Re-fetch macvlan to get all properties/attributes
|
||||
contMacvlan, err := netlink.LinkByName(ifName)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to refetch macvlan %q: %v", ifName, err)
|
||||
}
|
||||
macvlan.Mac = contMacvlan.Attrs().HardwareAddr.String()
|
||||
macvlan.Sandbox = netns.Path()
|
||||
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return macvlan, nil
|
||||
}
|
||||
|
||||
func cmdAdd(args *skel.CmdArgs) error {
|
||||
n, err := loadConf(args.StdinData)
|
||||
n, cniVersion, err := loadConf(args.StdinData)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@ -137,7 +155,8 @@ func cmdAdd(args *skel.CmdArgs) error {
|
||||
}
|
||||
defer netns.Close()
|
||||
|
||||
if err = createMacvlan(n, args.IfName, netns); err != nil {
|
||||
macvlanInterface, err := createMacvlan(n, args.IfName, netns)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@ -146,32 +165,60 @@ func cmdAdd(args *skel.CmdArgs) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
result, err := current.GetResult(r)
|
||||
// Convert whatever the IPAM result was into the current Result type
|
||||
result, err := current.NewResultFromResult(r)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if result.IP4 == nil {
|
||||
return errors.New("IPAM plugin returned missing IPv4 config")
|
||||
if len(result.IPs) == 0 {
|
||||
return errors.New("IPAM plugin returned missing IP config")
|
||||
}
|
||||
result.Interfaces = []*current.Interface{macvlanInterface}
|
||||
|
||||
var firstV4Addr net.IP
|
||||
for _, ipc := range result.IPs {
|
||||
// All addresses apply to the container macvlan interface
|
||||
ipc.Interface = 0
|
||||
|
||||
if ipc.Address.IP.To4() != nil && firstV4Addr == nil {
|
||||
firstV4Addr = ipc.Address.IP
|
||||
}
|
||||
}
|
||||
|
||||
err = netns.Do(func(_ ns.NetNS) error {
|
||||
if err := ip.SetHWAddrByIP(args.IfName, result.IP4.IP.IP, nil /* TODO IPv6 */); err != nil {
|
||||
if firstV4Addr != nil {
|
||||
err = netns.Do(func(_ ns.NetNS) error {
|
||||
if err := ip.SetHWAddrByIP(args.IfName, firstV4Addr, nil /* TODO IPv6 */); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return ipam.ConfigureIface(args.IfName, result)
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return ipam.ConfigureIface(args.IfName, result)
|
||||
// Re-fetch macvlan interface as its MAC address may have changed
|
||||
err = netns.Do(func(_ ns.NetNS) error {
|
||||
link, err := netlink.LinkByName(args.IfName)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to re-fetch macvlan interface: %v", err)
|
||||
}
|
||||
macvlanInterface.Mac = link.Attrs().HardwareAddr.String()
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
result.DNS = n.DNS
|
||||
return result.Print()
|
||||
|
||||
return types.PrintResult(result, cniVersion)
|
||||
}
|
||||
|
||||
func cmdDel(args *skel.CmdArgs) error {
|
||||
n, err := loadConf(args.StdinData)
|
||||
n, _, err := loadConf(args.StdinData)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@ -191,5 +238,5 @@ func cmdDel(args *skel.CmdArgs) error {
|
||||
}
|
||||
|
||||
func main() {
|
||||
skel.PluginMain(cmdAdd, cmdDel, version.Legacy)
|
||||
skel.PluginMain(cmdAdd, cmdDel, version.All)
|
||||
}
|
||||
|
@ -16,11 +16,14 @@ package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net"
|
||||
"syscall"
|
||||
|
||||
"github.com/containernetworking/cni/pkg/ns"
|
||||
"github.com/containernetworking/cni/pkg/skel"
|
||||
"github.com/containernetworking/cni/pkg/testutils"
|
||||
"github.com/containernetworking/cni/pkg/types"
|
||||
"github.com/containernetworking/cni/pkg/types/current"
|
||||
"github.com/containernetworking/cni/pkg/utils/hwaddr"
|
||||
|
||||
"github.com/vishvananda/netlink"
|
||||
@ -64,7 +67,7 @@ var _ = Describe("macvlan Operations", func() {
|
||||
It("creates an macvlan link in a non-default namespace", func() {
|
||||
conf := &NetConf{
|
||||
NetConf: types.NetConf{
|
||||
CNIVersion: "0.2.0",
|
||||
CNIVersion: "0.3.0",
|
||||
Name: "testConfig",
|
||||
Type: "macvlan",
|
||||
},
|
||||
@ -80,7 +83,7 @@ var _ = Describe("macvlan Operations", func() {
|
||||
err = originalNS.Do(func(ns.NetNS) error {
|
||||
defer GinkgoRecover()
|
||||
|
||||
err = createMacvlan(conf, "foobar0", targetNs)
|
||||
_, err = createMacvlan(conf, "foobar0", targetNs)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
return nil
|
||||
})
|
||||
@ -102,7 +105,7 @@ var _ = Describe("macvlan Operations", func() {
|
||||
const IFNAME = "macvl0"
|
||||
|
||||
conf := fmt.Sprintf(`{
|
||||
"cniVersion": "0.2.0",
|
||||
"cniVersion": "0.3.0",
|
||||
"name": "mynet",
|
||||
"type": "macvlan",
|
||||
"master": "%s",
|
||||
@ -123,14 +126,21 @@ var _ = Describe("macvlan Operations", func() {
|
||||
StdinData: []byte(conf),
|
||||
}
|
||||
|
||||
// Make sure macvlan link exists in the target namespace
|
||||
var result *current.Result
|
||||
err = originalNS.Do(func(ns.NetNS) error {
|
||||
defer GinkgoRecover()
|
||||
|
||||
_, _, err := testutils.CmdAddWithResult(targetNs.Path(), IFNAME, []byte(conf), func() error {
|
||||
r, _, err := testutils.CmdAddWithResult(targetNs.Path(), IFNAME, []byte(conf), func() error {
|
||||
return cmdAdd(args)
|
||||
})
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
result, err = current.GetResult(r)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
Expect(len(result.Interfaces)).To(Equal(1))
|
||||
Expect(result.Interfaces[0].Name).To(Equal(IFNAME))
|
||||
Expect(len(result.IPs)).To(Equal(1))
|
||||
return nil
|
||||
})
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
@ -143,9 +153,16 @@ var _ = Describe("macvlan Operations", func() {
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(link.Attrs().Name).To(Equal(IFNAME))
|
||||
|
||||
hwAddr := fmt.Sprintf("%s", link.Attrs().HardwareAddr)
|
||||
Expect(hwAddr).To(HavePrefix(hwaddr.PrivateMACPrefixString))
|
||||
hwaddrString := fmt.Sprintf("%s", link.Attrs().HardwareAddr)
|
||||
Expect(hwaddrString).To(HavePrefix(hwaddr.PrivateMACPrefixString))
|
||||
|
||||
hwaddr, err := net.ParseMAC(result.Interfaces[0].Mac)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(link.Attrs().HardwareAddr).To(Equal(hwaddr))
|
||||
|
||||
addrs, err := netlink.AddrList(link, syscall.AF_INET)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(len(addrs)).To(Equal(1))
|
||||
return nil
|
||||
})
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
@ -47,7 +47,7 @@ type NetConf struct {
|
||||
MTU int `json:"mtu"`
|
||||
}
|
||||
|
||||
func setupContainerVeth(netns, ifName string, mtu int, pr *current.Result) (string, error) {
|
||||
func setupContainerVeth(netns, ifName string, mtu int, pr *current.Result) (*current.Interface, *current.Interface, error) {
|
||||
// The IPAM result will be something like IP=192.168.3.5/24, GW=192.168.3.1.
|
||||
// What we want is really a point-to-point link but veth does not support IFF_POINTOPONT.
|
||||
// Next best thing would be to let it ARP but set interface to 192.168.3.5/32 and
|
||||
@ -59,104 +59,139 @@ func setupContainerVeth(netns, ifName string, mtu int, pr *current.Result) (stri
|
||||
// "192.168.3.1/32 dev $ifName" and "192.168.3.0/24 via 192.168.3.1 dev $ifName".
|
||||
// In other words we force all traffic to ARP via the gateway except for GW itself.
|
||||
|
||||
var hostVethName string
|
||||
hostInterface := ¤t.Interface{}
|
||||
containerInterface := ¤t.Interface{}
|
||||
|
||||
err := ns.WithNetNSPath(netns, func(hostNS ns.NetNS) error {
|
||||
hostVeth, _, err := ip.SetupVeth(ifName, mtu, hostNS)
|
||||
hostVeth, contVeth, err := ip.SetupVeth(ifName, mtu, hostNS)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
hostInterface.Name = hostVeth.Attrs().Name
|
||||
hostInterface.Mac = hostVeth.Attrs().HardwareAddr.String()
|
||||
containerInterface.Name = contVeth.Attrs().Name
|
||||
containerInterface.Mac = contVeth.Attrs().HardwareAddr.String()
|
||||
|
||||
hostNS.Do(func(_ ns.NetNS) error {
|
||||
hostVethName = hostVeth.Attrs().Name
|
||||
if err := ip.SetHWAddrByIP(hostVethName, pr.IP4.IP.IP, nil /* TODO IPv6 */); err != nil {
|
||||
return fmt.Errorf("failed to set hardware addr by IP: %v", err)
|
||||
var firstV4Addr net.IP
|
||||
for _, ipc := range pr.IPs {
|
||||
// All addresses apply to the container veth interface
|
||||
ipc.Interface = 1
|
||||
|
||||
if ipc.Address.IP.To4() != nil && firstV4Addr == nil {
|
||||
firstV4Addr = ipc.Address.IP
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
pr.Interfaces = []*current.Interface{hostInterface, containerInterface}
|
||||
|
||||
if firstV4Addr != nil {
|
||||
err = hostNS.Do(func(_ ns.NetNS) error {
|
||||
hostVethName := hostVeth.Attrs().Name
|
||||
if err := ip.SetHWAddrByIP(hostVethName, firstV4Addr, nil /* TODO IPv6 */); err != nil {
|
||||
return fmt.Errorf("failed to set hardware addr by IP: %v", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if err = ipam.ConfigureIface(ifName, pr); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
contVeth, err := netlink.LinkByName(ifName)
|
||||
if err := ip.SetHWAddrByIP(contVeth.Attrs().Name, firstV4Addr, nil /* TODO IPv6 */); err != nil {
|
||||
return fmt.Errorf("failed to set hardware addr by IP: %v", err)
|
||||
}
|
||||
|
||||
// Re-fetch container veth to update attributes
|
||||
contVeth, err = netlink.LinkByName(ifName)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to look up %q: %v", ifName, err)
|
||||
}
|
||||
|
||||
if err := ip.SetHWAddrByIP(contVeth.Attrs().Name, pr.IP4.IP.IP, nil /* TODO IPv6 */); err != nil {
|
||||
return fmt.Errorf("failed to set hardware addr by IP: %v", err)
|
||||
}
|
||||
|
||||
// Delete the route that was automatically added
|
||||
route := netlink.Route{
|
||||
LinkIndex: contVeth.Attrs().Index,
|
||||
Dst: &net.IPNet{
|
||||
IP: pr.IP4.IP.IP.Mask(pr.IP4.IP.Mask),
|
||||
Mask: pr.IP4.IP.Mask,
|
||||
},
|
||||
Scope: netlink.SCOPE_NOWHERE,
|
||||
}
|
||||
|
||||
if err := netlink.RouteDel(&route); err != nil {
|
||||
return fmt.Errorf("failed to delete route %v: %v", route, err)
|
||||
}
|
||||
|
||||
for _, r := range []netlink.Route{
|
||||
{
|
||||
for _, ipc := range pr.IPs {
|
||||
// Delete the route that was automatically added
|
||||
route := netlink.Route{
|
||||
LinkIndex: contVeth.Attrs().Index,
|
||||
Dst: &net.IPNet{
|
||||
IP: pr.IP4.Gateway,
|
||||
Mask: net.CIDRMask(32, 32),
|
||||
IP: ipc.Address.IP.Mask(ipc.Address.Mask),
|
||||
Mask: ipc.Address.Mask,
|
||||
},
|
||||
Scope: netlink.SCOPE_LINK,
|
||||
Src: pr.IP4.IP.IP,
|
||||
},
|
||||
{
|
||||
LinkIndex: contVeth.Attrs().Index,
|
||||
Dst: &net.IPNet{
|
||||
IP: pr.IP4.IP.IP.Mask(pr.IP4.IP.Mask),
|
||||
Mask: pr.IP4.IP.Mask,
|
||||
Scope: netlink.SCOPE_NOWHERE,
|
||||
}
|
||||
|
||||
if err := netlink.RouteDel(&route); err != nil {
|
||||
return fmt.Errorf("failed to delete route %v: %v", route, err)
|
||||
}
|
||||
|
||||
for _, r := range []netlink.Route{
|
||||
netlink.Route{
|
||||
LinkIndex: contVeth.Attrs().Index,
|
||||
Dst: &net.IPNet{
|
||||
IP: ipc.Gateway,
|
||||
Mask: net.CIDRMask(32, 32),
|
||||
},
|
||||
Scope: netlink.SCOPE_LINK,
|
||||
Src: ipc.Address.IP,
|
||||
},
|
||||
Scope: netlink.SCOPE_UNIVERSE,
|
||||
Gw: pr.IP4.Gateway,
|
||||
Src: pr.IP4.IP.IP,
|
||||
},
|
||||
} {
|
||||
if err := netlink.RouteAdd(&r); err != nil {
|
||||
return fmt.Errorf("failed to add route %v: %v", r, err)
|
||||
netlink.Route{
|
||||
LinkIndex: contVeth.Attrs().Index,
|
||||
Dst: &net.IPNet{
|
||||
IP: ipc.Address.IP.Mask(ipc.Address.Mask),
|
||||
Mask: ipc.Address.Mask,
|
||||
},
|
||||
Scope: netlink.SCOPE_UNIVERSE,
|
||||
Gw: ipc.Gateway,
|
||||
Src: ipc.Address.IP,
|
||||
},
|
||||
} {
|
||||
if err := netlink.RouteAdd(&r); err != nil {
|
||||
return fmt.Errorf("failed to add route %v: %v", r, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
return hostVethName, err
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
return hostInterface, containerInterface, nil
|
||||
}
|
||||
|
||||
func setupHostVeth(vethName string, ipConf *current.IPConfig) error {
|
||||
func setupHostVeth(vethName string, result *current.Result) error {
|
||||
// hostVeth moved namespaces and may have a new ifindex
|
||||
veth, err := netlink.LinkByName(vethName)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to lookup %q: %v", vethName, err)
|
||||
}
|
||||
|
||||
// TODO(eyakubovich): IPv6
|
||||
ipn := &net.IPNet{
|
||||
IP: ipConf.Gateway,
|
||||
Mask: net.CIDRMask(32, 32),
|
||||
}
|
||||
addr := &netlink.Addr{IPNet: ipn, Label: ""}
|
||||
if err = netlink.AddrAdd(veth, addr); err != nil {
|
||||
return fmt.Errorf("failed to add IP addr (%#v) to veth: %v", ipn, err)
|
||||
}
|
||||
for _, ipc := range result.IPs {
|
||||
maskLen := 128
|
||||
if ipc.Address.IP.To4() != nil {
|
||||
maskLen = 32
|
||||
}
|
||||
|
||||
ipn = &net.IPNet{
|
||||
IP: ipConf.IP.IP,
|
||||
Mask: net.CIDRMask(32, 32),
|
||||
}
|
||||
// dst happens to be the same as IP/net of host veth
|
||||
if err = ip.AddHostRoute(ipn, nil, veth); err != nil && !os.IsExist(err) {
|
||||
return fmt.Errorf("failed to add route on host: %v", err)
|
||||
ipn := &net.IPNet{
|
||||
IP: ipc.Gateway,
|
||||
Mask: net.CIDRMask(maskLen, maskLen),
|
||||
}
|
||||
addr := &netlink.Addr{IPNet: ipn, Label: ""}
|
||||
if err = netlink.AddrAdd(veth, addr); err != nil {
|
||||
return fmt.Errorf("failed to add IP addr (%#v) to veth: %v", ipn, err)
|
||||
}
|
||||
|
||||
ipn = &net.IPNet{
|
||||
IP: ipc.Address.IP,
|
||||
Mask: net.CIDRMask(maskLen, maskLen),
|
||||
}
|
||||
// dst happens to be the same as IP/net of host veth
|
||||
if err = ip.AddHostRoute(ipn, nil, veth); err != nil && !os.IsExist(err) {
|
||||
return fmt.Errorf("failed to add route on host: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
@ -177,33 +212,39 @@ func cmdAdd(args *skel.CmdArgs) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
result, err := current.GetResult(r)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if result.IP4 == nil {
|
||||
return errors.New("IPAM plugin returned missing IPv4 config")
|
||||
}
|
||||
|
||||
hostVethName, err := setupContainerVeth(args.Netns, args.IfName, conf.MTU, result)
|
||||
// Convert whatever the IPAM result was into the current Result type
|
||||
result, err := current.NewResultFromResult(r)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err = setupHostVeth(hostVethName, result.IP4); err != nil {
|
||||
if len(result.IPs) == 0 {
|
||||
return errors.New("IPAM plugin returned missing IP config")
|
||||
}
|
||||
|
||||
hostInterface, containerInterface, err := setupContainerVeth(args.Netns, args.IfName, conf.MTU, result)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err = setupHostVeth(hostInterface.Name, result); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if conf.IPMasq {
|
||||
chain := utils.FormatChainName(conf.Name, args.ContainerID)
|
||||
comment := utils.FormatComment(conf.Name, args.ContainerID)
|
||||
if err = ip.SetupIPMasq(&result.IP4.IP, chain, comment); err != nil {
|
||||
return err
|
||||
for _, ipc := range result.IPs {
|
||||
if err = ip.SetupIPMasq(&ipc.Address, chain, comment); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
result.DNS = conf.DNS
|
||||
return result.Print()
|
||||
result.Interfaces = []*current.Interface{hostInterface, containerInterface}
|
||||
|
||||
return types.PrintResult(result, conf.CNIVersion)
|
||||
}
|
||||
|
||||
func cmdDel(args *skel.CmdArgs) error {
|
||||
@ -242,5 +283,5 @@ func cmdDel(args *skel.CmdArgs) error {
|
||||
}
|
||||
|
||||
func main() {
|
||||
skel.PluginMain(cmdAdd, cmdDel, version.Legacy)
|
||||
skel.PluginMain(cmdAdd, cmdDel, version.All)
|
||||
}
|
||||
|
@ -43,7 +43,7 @@ var _ = Describe("ptp Operations", func() {
|
||||
const IFNAME = "ptp0"
|
||||
|
||||
conf := `{
|
||||
"cniVersion": "0.2.0",
|
||||
"cniVersion": "0.3.0",
|
||||
"name": "mynet",
|
||||
"type": "ptp",
|
||||
"ipMasq": true,
|
||||
|
@ -257,5 +257,5 @@ func cmdDel(args *skel.CmdArgs) error {
|
||||
}
|
||||
|
||||
func main() {
|
||||
skel.PluginMain(cmdAdd, cmdDel, version.Legacy)
|
||||
skel.PluginMain(cmdAdd, cmdDel, version.All)
|
||||
}
|
||||
|
@ -80,5 +80,5 @@ func cmdDel(args *skel.CmdArgs) error {
|
||||
}
|
||||
|
||||
func main() {
|
||||
skel.PluginMain(cmdAdd, cmdDel, version.Legacy)
|
||||
skel.PluginMain(cmdAdd, cmdDel, version.All)
|
||||
}
|
||||
|
@ -142,7 +142,7 @@ func debugBehavior(args *skel.CmdArgs, command string) error {
|
||||
}
|
||||
|
||||
func debugGetSupportedVersions(stdinData []byte) []string {
|
||||
vers := []string{"0.-42.0", "0.1.0", "0.2.0"}
|
||||
vers := []string{"0.-42.0", "0.1.0", "0.2.0", "0.3.0"}
|
||||
cniArgs := os.Getenv("CNI_ARGS")
|
||||
if cniArgs == "" {
|
||||
return vers
|
||||
|
@ -37,7 +37,7 @@ var _ = Describe("No-op plugin", func() {
|
||||
expectedCmdArgs skel.CmdArgs
|
||||
)
|
||||
|
||||
const reportResult = `{ "ip4": { "ip": "10.1.2.3/24" }, "dns": {} }`
|
||||
const reportResult = `{ "ips": [{ "version": "4", "address": "10.1.2.3/24" }], "dns": {} }`
|
||||
|
||||
BeforeEach(func() {
|
||||
debug = &noop_debug.Debug{
|
||||
@ -64,14 +64,14 @@ var _ = Describe("No-op plugin", func() {
|
||||
// Keep this last
|
||||
"CNI_ARGS=" + args,
|
||||
}
|
||||
cmd.Stdin = strings.NewReader(`{"some":"stdin-json", "cniVersion": "0.2.0"}`)
|
||||
cmd.Stdin = strings.NewReader(`{"some":"stdin-json", "cniVersion": "0.3.0"}`)
|
||||
expectedCmdArgs = skel.CmdArgs{
|
||||
ContainerID: "some-container-id",
|
||||
Netns: "/some/netns/path",
|
||||
IfName: "some-eth0",
|
||||
Args: args,
|
||||
Path: "/some/bin/path",
|
||||
StdinData: []byte(`{"some":"stdin-json", "cniVersion": "0.2.0"}`),
|
||||
StdinData: []byte(`{"some":"stdin-json", "cniVersion": "0.3.0"}`),
|
||||
}
|
||||
})
|
||||
|
||||
@ -102,15 +102,15 @@ var _ = Describe("No-op plugin", func() {
|
||||
|
||||
cmd.Stdin = strings.NewReader(`{
|
||||
"some":"stdin-json",
|
||||
"cniVersion": "0.2.0",
|
||||
"cniVersion": "0.3.0",
|
||||
"prevResult": {
|
||||
"ip4": {"ip": "10.1.2.15/24"}
|
||||
"ips": [{"version": "4", "address": "10.1.2.15/24"}]
|
||||
}
|
||||
}`)
|
||||
session, err := gexec.Start(cmd, GinkgoWriter, GinkgoWriter)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Eventually(session).Should(gexec.Exit(0))
|
||||
Expect(session.Out.Contents()).To(MatchJSON(`{"ip4": {"ip": "10.1.2.15/24"}, "dns": {}}`))
|
||||
Expect(session.Out.Contents()).To(MatchJSON(`{"ips": [{"version": "4", "address": "10.1.2.15/24"}], "dns": {}}`))
|
||||
})
|
||||
|
||||
It("injects DNS into previous result when ReportResult is INJECT-DNS", func() {
|
||||
@ -119,9 +119,9 @@ var _ = Describe("No-op plugin", func() {
|
||||
|
||||
cmd.Stdin = strings.NewReader(`{
|
||||
"some":"stdin-json",
|
||||
"cniVersion": "0.2.0",
|
||||
"cniVersion": "0.3.0",
|
||||
"prevResult": {
|
||||
"ip4": {"ip": "10.1.2.3/24"},
|
||||
"ips": [{"version": "4", "address": "10.1.2.3/24"}],
|
||||
"dns": {}
|
||||
}
|
||||
}`)
|
||||
@ -130,7 +130,7 @@ var _ = Describe("No-op plugin", func() {
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Eventually(session).Should(gexec.Exit(0))
|
||||
Expect(session.Out.Contents()).To(MatchJSON(`{
|
||||
"ip4": {"ip": "10.1.2.3/24"},
|
||||
"ips": [{"version": "4", "address": "10.1.2.3/24"}],
|
||||
"dns": {"nameservers": ["1.2.3.4"]}
|
||||
}`))
|
||||
})
|
||||
@ -139,7 +139,7 @@ var _ = Describe("No-op plugin", func() {
|
||||
// Remove the DEBUG option from CNI_ARGS and regular args
|
||||
newArgs := "FOO=BAR"
|
||||
cmd.Env[len(cmd.Env)-1] = "CNI_ARGS=" + newArgs
|
||||
newStdin := fmt.Sprintf(`{"some":"stdin-json", "cniVersion": "0.2.0", "debugFile": "%s"}`, debugFileName)
|
||||
newStdin := fmt.Sprintf(`{"some":"stdin-json", "cniVersion": "0.3.0", "debugFile": "%s"}`, debugFileName)
|
||||
cmd.Stdin = strings.NewReader(newStdin)
|
||||
expectedCmdArgs.Args = newArgs
|
||||
expectedCmdArgs.StdinData = []byte(newStdin)
|
||||
|
Reference in New Issue
Block a user