Merge pull request #809 from squeed/bridge-refresh-mac

bridge: refresh host-veth mac after port add

.github/actions/retest-action/Dockerfile

@@ -1,4 +1,4 @@
-FROM alpine:3.10
+FROM alpine:3.17
 
 RUN apk add --no-cache curl jq
 

.github/actions/retest-action/entrypoint.sh

@@ -27,10 +27,10 @@ curl --request GET \
     --header "authorization: Bearer ${GITHUB_TOKEN}" \
     --header "content-type: application/json" | jq '.workflow_runs | max_by(.run_number)' > run.json
 
-RERUN_URL=$(jq -r '.rerun_url' run.json)
+RUN_URL=$(jq -r '.rerun_url' run.json)
 
 curl --request POST \
-    --url "${RERUN_URL}" \
+    --url "${RUN_URL}/rerun-failed-jobs" \
     --header "authorization: Bearer ${GITHUB_TOKEN}" \
     --header "content-type: application/json"
 
@@ -42,4 +42,4 @@ curl --request POST \
     --header "authorization: Bearer ${GITHUB_TOKEN}" \
     --header "accept: application/vnd.github.squirrel-girl-preview+json" \
     --header "content-type: application/json" \
-    --data '{ "content" : "rocket" }'
+    --data '{ "content" : "rocket" }'

.github/dependabot.yml

@@ -0,0 +1,19 @@
+# To get started with Dependabot version updates, you'll need to specify which
+# package ecosystems to update and where the package manifests are located.
+# Please see the documentation for all configuration options:
+# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
+
+version: 2
+updates:
+  - package-ecosystem: "docker" # See documentation for possible values
+    directory: "/.github/actions/retest-action" # Location of package manifests
+    schedule:
+      interval: "weekly"
+  - package-ecosystem: "github-actions" # See documentation for possible values
+    directory: "/" # Location of package manifests
+    schedule:
+      interval: "weekly"
+  - package-ecosystem: "gomod" # See documentation for possible values
+    directory: "/" # Location of package manifests
+    schedule:
+      interval: "weekly"

.github/workflows/commands.yml

@@ -9,7 +9,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Re-Test Action
         uses: ./.github/actions/retest-action

.github/workflows/stale.yml

@@ -0,0 +1,13 @@
+name: 'Close stale issues and PRs'
+on:
+  schedule:
+    - cron: '30 1 * * *'
+
+jobs:
+  stale:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/stale@v7
+        with:
+          stale-pr-message: 'This PR has been untouched for too long without an update. It will be closed in 7 days.'
+          exempt-issue-labels: 'keep'

.github/workflows/test.yaml

@@ -4,8 +4,8 @@ name: test
 on: ["push", "pull_request"]
 
 env:
-  GO_VERSION: "1.15"
-  LINUX_ARCHES: "amd64 386 arm arm64 s390x mips64le ppc64le"
+  GO_VERSION: "1.18"
+  LINUX_ARCHES: "amd64 386 arm arm64 s390x mips64le ppc64le riscv64"
 
 jobs:
   build:
@@ -13,10 +13,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: setup go
-        uses: actions/setup-go@v1
+        uses: actions/setup-go@v3
         with:
           go-version: ${{ env.GO_VERSION }}
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
       - name: Build on all supported architectures
         run: |
@@ -31,11 +31,22 @@ jobs:
     name: Run tests on Linux amd64
     runs-on: ubuntu-latest
     steps:
+      - name: Install kernel module
+        run: |
+          sudo apt-get update
+          sudo apt-get install linux-modules-extra-$(uname -r)
+      - name: Install nftables
+        run: sudo apt-get install nftables
+
       - name: setup go
-        uses: actions/setup-go@v1
+        uses: actions/setup-go@v3
         with:
           go-version: ${{ env.GO_VERSION }}
-      - uses: actions/checkout@v2
+      - name: Set up Go for root
+        run: |
+          sudo ln -sf `which go` `sudo which go` || true
+          sudo go version
+      - uses: actions/checkout@v3
 
       - name: Install test binaries
         env:
@@ -61,9 +72,9 @@ jobs:
     runs-on: windows-latest
     steps:
       - name: setup go
-        uses: actions/setup-go@v1
+        uses: actions/setup-go@v3
         with:
           go-version: ${{ env.GO_VERSION }}
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: test
         run: bash ./test_windows.sh

OWNERS.md

@@ -1,10 +1,10 @@
 # Owners
 This is the official list of the CNI network plugins owners:
 - Bruce Ma <brucema19901024@gmail.com> (@mars1024)
-- Bryan Boreham <bryan@weave.works> (@bboreham)
 - Casey Callendrello <cdc@redhat.com> (@squeed)
 - Dan Williams <dcbw@redhat.com> (@dcbw)
 - Gabe Rosenhouse <grosenhouse@pivotal.io> (@rosenhouse)
 - Matt Dupre <matt@tigera.io> (@matthewdupre)
 - Michael Cambria <mcambria@redhat.com> (@mccv1r0)
 - Piotr Skarmuk <piotr.skarmuk@gmail.com> (@jellonek)
+- Michael Zappa <michael.zappa@gmail.com> (@MikeZappa87)

README.md

@@ -1,6 +1,6 @@
-[![Build Status](https://travis-ci.org/containernetworking/plugins.svg?branch=master)](https://travis-ci.org/containernetworking/plugins)
+[![test](https://github.com/containernetworking/plugins/actions/workflows/test.yaml/badge.svg)](https://github.com/containernetworking/plugins/actions/workflows/test.yaml?query=branch%3Amaster)
 
-# plugins
+# Plugins
 Some CNI network plugins, maintained by the containernetworking team. For more information, see the [CNI website](https://www.cni.dev).
 
 Read [CONTRIBUTING](CONTRIBUTING.md) for build and test instructions.
@@ -14,16 +14,16 @@ Read [CONTRIBUTING](CONTRIBUTING.md) for build and test instructions.
 * `ptp`: Creates a veth pair.
 * `vlan`: Allocates a vlan device.
 * `host-device`: Move an already-existing device into a container.
-#### Windows: windows specific
+* `dummy`: Creates a new Dummy device in the container.
+#### Windows: Windows specific
 * `win-bridge`: Creates a bridge, adds the host and the container to it.
 * `win-overlay`: Creates an overlay interface to the container.
 ### IPAM: IP address allocation
 * `dhcp`: Runs a daemon on the host to make DHCP requests on behalf of the container
 * `host-local`: Maintains a local database of allocated IPs
-* `static`:  Allocate a static IPv4/IPv6 addresses to container and it's useful in debugging purpose.
+* `static`:  Allocate a single static IPv4/IPv6 address to container. It's useful in debugging purpose.
 
 ### Meta: other plugins
-* `flannel`: Generates an interface corresponding to a flannel config file
 * `tuning`: Tweaks sysctl parameters of an existing interface
 * `portmap`: An iptables-based portmapping plugin. Maps ports from the host's address space to the container.
 * `bandwidth`: Allows bandwidth-limiting through use of traffic control tbf (ingress/egress).

go.mod

@@ -1,29 +1,41 @@
 module github.com/containernetworking/plugins
 
-go 1.14
+go 1.17
 
 require (
-	github.com/Microsoft/go-winio v0.4.11 // indirect
-	github.com/Microsoft/hcsshim v0.8.6
-	github.com/alexflint/go-filemutex v0.0.0-20171022225611-72bdc8eae2ae
-	github.com/buger/jsonparser v0.0.0-20180808090653-f4dd9f5a6b44
-	github.com/containernetworking/cni v0.8.1
-	github.com/coreos/go-iptables v0.5.0
-	github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7
+	github.com/Microsoft/hcsshim v0.9.6
+	github.com/alexflint/go-filemutex v1.2.0
+	github.com/buger/jsonparser v1.1.1
+	github.com/containernetworking/cni v1.1.2
+	github.com/coreos/go-iptables v0.6.0
+	github.com/coreos/go-systemd/v22 v22.5.0
 	github.com/d2g/dhcp4 v0.0.0-20170904100407-a1d1b6c41b1c
 	github.com/d2g/dhcp4client v1.0.0
 	github.com/d2g/dhcp4server v0.0.0-20181031114812-7d4a0a7f59a5
-	github.com/d2g/hardwareaddr v0.0.0-20190221164911-e7d9fbe030e4 // indirect
-	github.com/godbus/dbus v0.0.0-20180201030542-885f9cc04c9c
-	github.com/j-keck/arping v0.0.0-20160618110441-2cf9dc699c56
-	github.com/mattn/go-shellwords v1.0.3
-	github.com/onsi/ginkgo v1.12.1
-	github.com/onsi/gomega v1.10.3
-	github.com/safchain/ethtool v0.0.0-20190326074333-42ed695e3de8
-	github.com/sirupsen/logrus v1.0.6 // indirect
-	github.com/stretchr/testify v1.3.0 // indirect
-	github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852
-	golang.org/x/sys v0.0.0-20201117170446-d9b008d0a637
-	gopkg.in/airbrake/gobrake.v2 v2.0.9 // indirect
-	gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2 // indirect
+	github.com/godbus/dbus/v5 v5.1.0
+	github.com/mattn/go-shellwords v1.0.12
+	github.com/networkplumbing/go-nft v0.2.0
+	github.com/onsi/ginkgo v1.16.5
+	github.com/onsi/gomega v1.24.2
+	github.com/safchain/ethtool v0.2.0
+	github.com/vishvananda/netlink v1.2.1-beta.2
+	golang.org/x/sys v0.4.0
+)
+
+require (
+	github.com/Microsoft/go-winio v0.4.17 // indirect
+	github.com/containerd/cgroups v1.0.1 // indirect
+	github.com/fsnotify/fsnotify v1.4.9 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
+	github.com/google/go-cmp v0.5.9 // indirect
+	github.com/nxadm/tail v1.4.8 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/sirupsen/logrus v1.8.1 // indirect
+	github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f // indirect
+	go.opencensus.io v0.22.3 // indirect
+	golang.org/x/net v0.4.0 // indirect
+	golang.org/x/text v0.5.0 // indirect
+	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

pkg/hns/endpoint_windows.go

@@ -23,7 +23,8 @@ import (
 	"github.com/Microsoft/hcsshim/hcn"
 
 	"github.com/containernetworking/cni/pkg/types"
-	"github.com/containernetworking/cni/pkg/types/current"
+	current "github.com/containernetworking/cni/pkg/types/100"
+
 	"github.com/containernetworking/plugins/pkg/errors"
 )
 
@@ -38,9 +39,10 @@ type EndpointInfo struct {
 	NetworkId    string
 	Gateway      net.IP
 	IpAddress    net.IP
+	MacAddress  string
 }
 
-// GetSandboxContainerID returns the sandbox ID of this pod
+// GetSandboxContainerID returns the sandbox ID of this pod.
 func GetSandboxContainerID(containerID string, netNs string) string {
 	if len(netNs) != 0 && netNs != pauseContainerNetNS {
 		splits := strings.SplitN(netNs, ":", 2)
@@ -52,7 +54,7 @@ func GetSandboxContainerID(containerID string, netNs string) string {
 	return containerID
 }
 
-// short function so we know when to return "" for a string
+// GetIpString returns the given IP as a string.
 func GetIpString(ip *net.IP) string {
 	if len(*ip) == 0 {
 		return ""
@@ -61,222 +63,136 @@ func GetIpString(ip *net.IP) string {
 	}
 }
 
-func GenerateHnsEndpoint(epInfo *EndpointInfo, n *NetConf) (*hcsshim.HNSEndpoint, error) {
-	// run the IPAM plugin and get back the config to apply
-	hnsEndpoint, err := hcsshim.GetHNSEndpointByName(epInfo.EndpointName)
-	if err != nil && !hcsshim.IsNotExist(err) {
-		return nil, errors.Annotatef(err, "failed to get endpoint %q", epInfo.EndpointName)
+// GetDefaultDestinationPrefix returns the default destination prefix according to the given IP type.
+func GetDefaultDestinationPrefix(ip *net.IP) string {
+	destinationPrefix := "0.0.0.0/0"
+	if ip.To4() == nil {
+		destinationPrefix = "::/0"
 	}
-
-	if hnsEndpoint != nil {
-		if hnsEndpoint.VirtualNetwork != epInfo.NetworkId {
-			_, err = hnsEndpoint.Delete()
-			if err != nil {
-				return nil, errors.Annotatef(err, "failed to delete endpoint %s", epInfo.EndpointName)
-			}
-			hnsEndpoint = nil
-		}
-	}
-
-	if n.LoopbackDSR {
-		n.ApplyLoopbackDSR(&epInfo.IpAddress)
-	}
-	if hnsEndpoint == nil {
-		hnsEndpoint = &hcsshim.HNSEndpoint{
-			Name:           epInfo.EndpointName,
-			VirtualNetwork: epInfo.NetworkId,
-			DNSServerList:  strings.Join(epInfo.DNS.Nameservers, ","),
-			DNSSuffix:      strings.Join(epInfo.DNS.Search, ","),
-			GatewayAddress: GetIpString(&epInfo.Gateway),
-			IPAddress:      epInfo.IpAddress,
-			Policies:       n.MarshalPolicies(),
-		}
-	}
-	return hnsEndpoint, nil
+	return destinationPrefix
 }
 
-func GenerateHcnEndpoint(epInfo *EndpointInfo, n *NetConf) (*hcn.HostComputeEndpoint, error) {
-	// run the IPAM plugin and get back the config to apply
-	hcnEndpoint, err := hcn.GetEndpointByName(epInfo.EndpointName)
-	if err != nil && !hcn.IsNotFoundError(err) {
-		return nil, errors.Annotatef(err, "failed to get endpoint %q", epInfo.EndpointName)
-	}
-
-	if hcnEndpoint != nil {
-		// If the endpont already exists, then we should return error unless
-		// the endpoint is based on a different network then delete
-		// should that fail return error
-		if !strings.EqualFold(hcnEndpoint.HostComputeNetwork, epInfo.NetworkId) {
-			err = hcnEndpoint.Delete()
-			if err != nil {
-				return nil, errors.Annotatef(err, "failed to delete endpoint %s", epInfo.EndpointName)
-			}
-		} else {
-			return nil, fmt.Errorf("endpoint %q already exits", epInfo.EndpointName)
-		}
-	}
-
-	if hcnEndpoint == nil {
-		routes := []hcn.Route{
-			{
-				NextHop:           GetIpString(&epInfo.Gateway),
-				DestinationPrefix: GetDefaultDestinationPrefix(&epInfo.Gateway),
-			},
-		}
-
-		hcnDns := hcn.Dns{
-			Search:     epInfo.DNS.Search,
-			ServerList: epInfo.DNS.Nameservers,
-		}
-
-		hcnIpConfig := hcn.IpConfig{
-			IpAddress: GetIpString(&epInfo.IpAddress),
-		}
-		ipConfigs := []hcn.IpConfig{hcnIpConfig}
-
-		if n.LoopbackDSR {
-			n.ApplyLoopbackDSR(&epInfo.IpAddress)
-		}
-		hcnEndpoint = &hcn.HostComputeEndpoint{
-			SchemaVersion:      hcn.Version{Major: 2},
-			Name:               epInfo.EndpointName,
-			HostComputeNetwork: epInfo.NetworkId,
-			Dns:                hcnDns,
-			Routes:             routes,
-			IpConfigurations:   ipConfigs,
-			Policies: func() []hcn.EndpointPolicy {
-				if n.HcnPolicyArgs == nil {
-					n.HcnPolicyArgs = []hcn.EndpointPolicy{}
-				}
-				return n.HcnPolicyArgs
-			}(),
-		}
-	}
-	return hcnEndpoint, nil
-}
-
-// ConstructEndpointName constructs enpointId which is used to identify an endpoint from HNS
-// There is a special consideration for netNs name here, which is required for Windows Server 1709
-// containerID is the Id of the container on which the endpoint is worked on
+// ConstructEndpointName constructs endpoint id which is used to identify an endpoint from HNS/HCN.
 func ConstructEndpointName(containerID string, netNs string, networkName string) string {
 	return GetSandboxContainerID(containerID, netNs) + "_" + networkName
 }
 
-// DeprovisionEndpoint removes an endpoint from the container by sending a Detach request to HNS
-// For shared endpoint, ContainerDetach is used
-// for removing the endpoint completely, HotDetachEndpoint is used
-func DeprovisionEndpoint(epName string, netns string, containerID string) error {
+// GenerateHnsEndpoint generates an HNSEndpoint with given info and config.
+func GenerateHnsEndpoint(epInfo *EndpointInfo, n *NetConf) (*hcsshim.HNSEndpoint, error) {
+	// run the IPAM plugin and get back the config to apply
+	hnsEndpoint, err := hcsshim.GetHNSEndpointByName(epInfo.EndpointName)
+	if err != nil && !hcsshim.IsNotExist(err) {
+		return nil, errors.Annotatef(err, "failed to get HNSEndpoint %s", epInfo.EndpointName)
+	}
+
+	if hnsEndpoint != nil {
+		if strings.EqualFold(hnsEndpoint.VirtualNetwork, epInfo.NetworkId) {
+			return nil, fmt.Errorf("HNSEndpoint %s is already existed", epInfo.EndpointName)
+		}
+		// remove endpoint if corrupted
+		if _, err = hnsEndpoint.Delete(); err != nil {
+			return nil, errors.Annotatef(err, "failed to delete corrupted HNSEndpoint %s", epInfo.EndpointName)
+		}
+	}
+
+	if n.LoopbackDSR {
+		n.ApplyLoopbackDSRPolicy(&epInfo.IpAddress)
+	}
+	hnsEndpoint = &hcsshim.HNSEndpoint{
+		Name:           epInfo.EndpointName,
+		VirtualNetwork: epInfo.NetworkId,
+		DNSServerList:  strings.Join(epInfo.DNS.Nameservers, ","),
+		DNSSuffix:      strings.Join(epInfo.DNS.Search, ","),
+		GatewayAddress: GetIpString(&epInfo.Gateway),
+		IPAddress:      epInfo.IpAddress,
+		Policies:       n.GetHNSEndpointPolicies(),
+	}
+	return hnsEndpoint, nil
+}
+
+// RemoveHnsEndpoint detaches the given name endpoint from container specified by containerID,
+// or removes the given name endpoint completely.
+func RemoveHnsEndpoint(epName string, netns string, containerID string) error {
 	if len(netns) == 0 {
 		return nil
 	}
 
 	hnsEndpoint, err := hcsshim.GetHNSEndpointByName(epName)
-
-	if hcsshim.IsNotExist(err) {
-		return nil
-	} else if err != nil {
+	if err != nil {
+		if hcsshim.IsNotExist(err) {
+			return nil
+		}
 		return errors.Annotatef(err, "failed to find HNSEndpoint %s", epName)
 	}
 
+	// for shared endpoint, detach it from the container
 	if netns != pauseContainerNetNS {
-		// Shared endpoint removal. Do not remove the endpoint.
-		hnsEndpoint.ContainerDetach(containerID)
+		_ = hnsEndpoint.ContainerDetach(containerID)
 		return nil
 	}
 
-	// Do not consider this as failure, else this would leak endpoints
-	hcsshim.HotDetachEndpoint(containerID, hnsEndpoint.Id)
-
-	// Do not return error
-	hnsEndpoint.Delete()
-
+	// for removing the endpoint completely, hot detach is used at first
+	_ = hcsshim.HotDetachEndpoint(containerID, hnsEndpoint.Id)
+	_, _ = hnsEndpoint.Delete()
 	return nil
 }
 
-type EndpointMakerFunc func() (*hcsshim.HNSEndpoint, error)
+type HnsEndpointMakerFunc func() (*hcsshim.HNSEndpoint, error)
 
-// ProvisionEndpoint provisions an endpoint to a container specified by containerID.
-// If an endpoint already exists, the endpoint is reused.
-// This call is idempotent
-func ProvisionEndpoint(epName string, expectedNetworkId string, containerID string, netns string, makeEndpoint EndpointMakerFunc) (*hcsshim.HNSEndpoint, error) {
-	// On the second add call we expect that the endpoint already exists. If it
-	// does not then we should return an error.
-	if netns != pauseContainerNetNS {
-		_, err := hcsshim.GetHNSEndpointByName(epName)
-		if err != nil {
+// AddHnsEndpoint attaches an HNSEndpoint to a container specified by containerID.
+func AddHnsEndpoint(epName string, expectedNetworkId string, containerID string, netns string, makeEndpoint HnsEndpointMakerFunc) (*hcsshim.HNSEndpoint, error) {
+	hnsEndpoint, err := hcsshim.GetHNSEndpointByName(epName)
+	if err != nil {
+		if !hcsshim.IsNotExist(err) {
 			return nil, errors.Annotatef(err, "failed to find HNSEndpoint %s", epName)
 		}
 	}
 
-	// check if endpoint already exists
-	createEndpoint := true
-	hnsEndpoint, err := hcsshim.GetHNSEndpointByName(epName)
-	if hnsEndpoint != nil && strings.EqualFold(hnsEndpoint.VirtualNetwork, expectedNetworkId) {
-		createEndpoint = false
+	// for shared endpoint, we expect that the endpoint already exists
+	if netns != pauseContainerNetNS {
+		if hnsEndpoint == nil {
+			return nil, errors.Annotatef(err, "failed to find HNSEndpoint %s", epName)
+		}
 	}
 
-	if createEndpoint {
-		if hnsEndpoint != nil {
-			if _, err = hnsEndpoint.Delete(); err != nil {
-				return nil, errors.Annotate(err, "failed to delete the stale HNSEndpoint")
+	// verify the existing endpoint is corrupted or not
+	if hnsEndpoint != nil {
+		if !strings.EqualFold(hnsEndpoint.VirtualNetwork, expectedNetworkId) {
+			if _, err := hnsEndpoint.Delete(); err != nil {
+				return nil, errors.Annotatef(err, "failed to delete corrupted HNSEndpoint %s", epName)
 			}
+			hnsEndpoint = nil
 		}
+	}
 
+	// create endpoint if not found
+	var isNewEndpoint bool
+	if hnsEndpoint == nil {
 		if hnsEndpoint, err = makeEndpoint(); err != nil {
 			return nil, errors.Annotate(err, "failed to make a new HNSEndpoint")
 		}
-
 		if hnsEndpoint, err = hnsEndpoint.Create(); err != nil {
 			return nil, errors.Annotate(err, "failed to create the new HNSEndpoint")
 		}
-
+		isNewEndpoint = true
 	}
 
-	// hot attach
+	// attach to container
 	if err := hcsshim.HotAttachEndpoint(containerID, hnsEndpoint.Id); err != nil {
-		if createEndpoint {
-			err := DeprovisionEndpoint(epName, netns, containerID)
-			if err != nil {
-				return nil, errors.Annotatef(err, "failed to Deprovsion after HotAttach failure")
+		if isNewEndpoint {
+			if err := RemoveHnsEndpoint(epName, netns, containerID); err != nil {
+				return nil, errors.Annotatef(err, "failed to remove the new HNSEndpoint %s after attaching container %s failure", hnsEndpoint.Id, containerID)
 			}
-		}
-		if hcsshim.ErrComputeSystemDoesNotExist == err {
+		} else if hcsshim.ErrComputeSystemDoesNotExist == err {
 			return hnsEndpoint, nil
 		}
-		return nil, err
+		return nil, errors.Annotatef(err, "failed to attach container %s to HNSEndpoint %s", containerID, hnsEndpoint.Id)
 	}
-
 	return hnsEndpoint, nil
 }
 
-type HcnEndpointMakerFunc func() (*hcn.HostComputeEndpoint, error)
-
-func AddHcnEndpoint(epName string, expectedNetworkId string, namespace string,
-	makeEndpoint HcnEndpointMakerFunc) (*hcn.HostComputeEndpoint, error) {
-
-	hcnEndpoint, err := makeEndpoint()
-	if err != nil {
-		return nil, errors.Annotate(err, "failed to make a new HNSEndpoint")
-	}
-
-	if hcnEndpoint, err = hcnEndpoint.Create(); err != nil {
-		return nil, errors.Annotate(err, "failed to create the new HNSEndpoint")
-	}
-
-	err = hcn.AddNamespaceEndpoint(namespace, hcnEndpoint.Id)
-	if err != nil {
-		err := RemoveHcnEndpoint(epName)
-		if err != nil {
-			return nil, errors.Annotatef(err, "failed to Remove Endpoint after AddNamespaceEndpoint failure")
-		}
-		return nil, errors.Annotate(err, "failed to Add endpoint to namespace")
-	}
-	return hcnEndpoint, nil
-
-}
-
-// ConstructResult constructs the CNI result for the endpoint
-func ConstructResult(hnsNetwork *hcsshim.HNSNetwork, hnsEndpoint *hcsshim.HNSEndpoint) (*current.Result, error) {
+// ConstructHnsResult constructs the CNI result for the HNSEndpoint.
+func ConstructHnsResult(hnsNetwork *hcsshim.HNSNetwork, hnsEndpoint *hcsshim.HNSEndpoint) (*current.Result, error) {
 	resultInterface := &current.Interface{
 		Name: hnsEndpoint.Name,
 		Mac:  hnsEndpoint.MacAddress,
@@ -286,51 +202,151 @@ func ConstructResult(hnsNetwork *hcsshim.HNSNetwork, hnsEndpoint *hcsshim.HNSEnd
 		return nil, errors.Annotatef(err, "failed to parse CIDR from %s", hnsNetwork.Subnets[0].AddressPrefix)
 	}
 
-	var ipVersion string
-	if ipv4 := hnsEndpoint.IPAddress.To4(); ipv4 != nil {
-		ipVersion = "4"
-	} else if ipv6 := hnsEndpoint.IPAddress.To16(); ipv6 != nil {
-		ipVersion = "6"
-	} else {
-		return nil, fmt.Errorf("IPAddress of HNSEndpoint %s isn't a valid ipv4 or ipv6 Address", hnsEndpoint.Name)
-	}
-
 	resultIPConfig := &current.IPConfig{
-		Version: ipVersion,
 		Address: net.IPNet{
 			IP:   hnsEndpoint.IPAddress,
 			Mask: ipSubnet.Mask},
 		Gateway: net.ParseIP(hnsEndpoint.GatewayAddress),
 	}
-	result := &current.Result{}
-	result.Interfaces = []*current.Interface{resultInterface}
-	result.IPs = []*current.IPConfig{resultIPConfig}
-	result.DNS = types.DNS{
-		Search:      strings.Split(hnsEndpoint.DNSSuffix, ","),
-		Nameservers: strings.Split(hnsEndpoint.DNSServerList, ","),
+	result := &current.Result{
+		CNIVersion: current.ImplementedSpecVersion,
+		Interfaces: []*current.Interface{resultInterface},
+		IPs:        []*current.IPConfig{resultIPConfig},
+		DNS: types.DNS{
+			Search:      strings.Split(hnsEndpoint.DNSSuffix, ","),
+			Nameservers: strings.Split(hnsEndpoint.DNSServerList, ","),
+		},
 	}
 
 	return result, nil
 }
 
-// This version follows the v2 workflow of removing the endpoint from the namespace and deleting it
+// GenerateHcnEndpoint generates a HostComputeEndpoint with given info and config.
+func GenerateHcnEndpoint(epInfo *EndpointInfo, n *NetConf) (*hcn.HostComputeEndpoint, error) {
+	// run the IPAM plugin and get back the config to apply
+	hcnEndpoint, err := hcn.GetEndpointByName(epInfo.EndpointName)
+	if err != nil && !hcn.IsNotFoundError(err) {
+		return nil, errors.Annotatef(err, "failed to get HostComputeEndpoint %s", epInfo.EndpointName)
+	}
+
+	// verify the existing endpoint is corrupted or not
+	if hcnEndpoint != nil {
+		if strings.EqualFold(hcnEndpoint.HostComputeNetwork, epInfo.NetworkId) {
+			return nil, fmt.Errorf("HostComputeNetwork %s is already existed", epInfo.EndpointName)
+		}
+		// remove endpoint if corrupted
+		if err := hcnEndpoint.Delete(); err != nil {
+			return nil, errors.Annotatef(err, "failed to delete corrupted HostComputeEndpoint %s", epInfo.EndpointName)
+		}
+	}
+
+	if n.LoopbackDSR {
+		n.ApplyLoopbackDSRPolicy(&epInfo.IpAddress)
+	}
+	hcnEndpoint = &hcn.HostComputeEndpoint{
+		SchemaVersion: hcn.SchemaVersion{
+			Major: 2,
+			Minor: 0,
+		},
+		Name:               epInfo.EndpointName,
+		MacAddress: epInfo.MacAddress,
+		HostComputeNetwork: epInfo.NetworkId,
+		Dns: hcn.Dns{
+			Domain:     epInfo.DNS.Domain,
+			Search:     epInfo.DNS.Search,
+			ServerList: epInfo.DNS.Nameservers,
+			Options:    epInfo.DNS.Options,
+		},
+		Routes: []hcn.Route{
+			{
+				NextHop:           GetIpString(&epInfo.Gateway),
+				DestinationPrefix: GetDefaultDestinationPrefix(&epInfo.Gateway),
+			},
+		},
+		IpConfigurations: []hcn.IpConfig{
+			{
+				IpAddress: GetIpString(&epInfo.IpAddress),
+			},
+		},
+		Policies: n.GetHostComputeEndpointPolicies(),
+	}
+	return hcnEndpoint, nil
+}
+
+// RemoveHcnEndpoint removes the given name endpoint from namespace.
 func RemoveHcnEndpoint(epName string) error {
 	hcnEndpoint, err := hcn.GetEndpointByName(epName)
-	if hcn.IsNotFoundError(err) {
-		return nil
-	} else if err != nil {
-		_ = fmt.Errorf("[win-cni] Failed to find endpoint %v, err:%v", epName, err)
-		return err
-	}
-	if hcnEndpoint != nil {
-		err = hcnEndpoint.Delete()
-		if err != nil {
-			return fmt.Errorf("[win-cni] Failed to delete endpoint %v, err:%v", epName, err)
+	if err != nil {
+		if hcn.IsNotFoundError(err) {
+			return nil
 		}
+		return errors.Annotatef(err, "failed to find HostComputeEndpoint %s", epName)
+	}
+	epNamespace, err := hcn.GetNamespaceByID(hcnEndpoint.HostComputeNamespace)
+	if err != nil && !hcn.IsNotFoundError(err) {
+		return errors.Annotatef(err, "failed to get HostComputeNamespace %s", epName)
+	}
+	if epNamespace != nil {
+		err = hcn.RemoveNamespaceEndpoint(hcnEndpoint.HostComputeNamespace, hcnEndpoint.Id)
+		if err != nil && !hcn.IsNotFoundError(err) {
+			return errors.Annotatef(err,"error removing endpoint: %s from namespace", epName)
+		}
+	}
+
+	err = hcnEndpoint.Delete()
+	if err != nil {
+		return errors.Annotatef(err, "failed to remove HostComputeEndpoint %s", epName)
 	}
 	return nil
 }
 
+type HcnEndpointMakerFunc func() (*hcn.HostComputeEndpoint, error)
+
+// AddHcnEndpoint attaches a HostComputeEndpoint to the given namespace.
+func AddHcnEndpoint(epName string, expectedNetworkId string, namespace string, makeEndpoint HcnEndpointMakerFunc) (*hcn.HostComputeEndpoint, error) {
+	hcnEndpoint, err := hcn.GetEndpointByName(epName)
+	if err != nil {
+		if !hcn.IsNotFoundError(err) {
+			return nil, errors.Annotatef(err, "failed to find HostComputeEndpoint %s", epName)
+		}
+	}
+
+	// verify the existing endpoint is corrupted or not
+	if hcnEndpoint != nil {
+		if !strings.EqualFold(hcnEndpoint.HostComputeNetwork, expectedNetworkId) {
+			if err := hcnEndpoint.Delete(); err != nil {
+				return nil, errors.Annotatef(err, "failed to delete corrupted HostComputeEndpoint %s", epName)
+			}
+			hcnEndpoint = nil
+		}
+	}
+
+	// create endpoint if not found
+	var isNewEndpoint bool
+	if hcnEndpoint == nil {
+		if hcnEndpoint, err = makeEndpoint(); err != nil {
+			return nil, errors.Annotate(err, "failed to make a new HostComputeEndpoint")
+		}
+		if hcnEndpoint, err = hcnEndpoint.Create(); err != nil {
+			return nil, errors.Annotate(err, "failed to create the new HostComputeEndpoint")
+		}
+		isNewEndpoint = true
+	}
+
+	// add to namespace
+	err = hcn.AddNamespaceEndpoint(namespace, hcnEndpoint.Id)
+	if err != nil {
+		if isNewEndpoint {
+			if err := RemoveHcnEndpoint(epName); err != nil {
+				return nil, errors.Annotatef(err, "failed to remove the new HostComputeEndpoint %s after adding HostComputeNamespace %s failure", epName, namespace)
+			}
+		}
+		return nil, errors.Annotatef(err, "failed to add HostComputeEndpoint %s to HostComputeNamespace %s", epName, namespace)
+	}
+	return hcnEndpoint, nil
+}
+
+// ConstructHcnResult constructs the CNI result for the HostComputeEndpoint.
 func ConstructHcnResult(hcnNetwork *hcn.HostComputeNetwork, hcnEndpoint *hcn.HostComputeEndpoint) (*current.Result, error) {
 	resultInterface := &current.Interface{
 		Name: hcnEndpoint.Name,
@@ -341,29 +357,23 @@ func ConstructHcnResult(hcnNetwork *hcn.HostComputeNetwork, hcnEndpoint *hcn.Hos
 		return nil, err
 	}
 
-	var ipVersion string
 	ipAddress := net.ParseIP(hcnEndpoint.IpConfigurations[0].IpAddress)
-	if ipv4 := ipAddress.To4(); ipv4 != nil {
-		ipVersion = "4"
-	} else if ipv6 := ipAddress.To16(); ipv6 != nil {
-		ipVersion = "6"
-	} else {
-		return nil, fmt.Errorf("[win-cni] The IPAddress of hnsEndpoint isn't a valid ipv4 or ipv6 Address.")
-	}
-
 	resultIPConfig := &current.IPConfig{
-		Version: ipVersion,
 		Address: net.IPNet{
 			IP:   ipAddress,
 			Mask: ipSubnet.Mask},
 		Gateway: net.ParseIP(hcnEndpoint.Routes[0].NextHop),
 	}
-	result := &current.Result{}
-	result.Interfaces = []*current.Interface{resultInterface}
-	result.IPs = []*current.IPConfig{resultIPConfig}
-	result.DNS = types.DNS{
-		Search:      hcnEndpoint.Dns.Search,
-		Nameservers: hcnEndpoint.Dns.ServerList,
+	result := &current.Result{
+		CNIVersion: current.ImplementedSpecVersion,
+		Interfaces: []*current.Interface{resultInterface},
+		IPs:        []*current.IPConfig{resultIPConfig},
+		DNS: types.DNS{
+			Search:      hcnEndpoint.Dns.Search,
+			Nameservers: hcnEndpoint.Dns.ServerList,
+			Options:     hcnEndpoint.Dns.Options,
+			Domain:      hcnEndpoint.Dns.Domain,
+		},
 	}
 
 	return result, nil

pkg/hns/netconf_suite_windows_test.go

@@ -20,7 +20,7 @@ import (
 	"testing"
 )
 
-func TestHns(t *testing.T) {
+func TestNetConf(t *testing.T) {
 	RegisterFailHandler(Fail)
-	RunSpecs(t, "HNS NetConf Suite")
+	RunSpecs(t, "NetConf Suite")
 }

pkg/hns/netconf_windows.go

@@ -17,9 +17,10 @@ package hns
 import (
 	"bytes"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"net"
-
+	"strconv"
 	"strings"
 
 	"github.com/Microsoft/hcsshim/hcn"
@@ -30,16 +31,16 @@ import (
 // NetConf is the CNI spec
 type NetConf struct {
 	types.NetConf
-	// ApiVersion is either 1 or 2, which specifies which hns APIs to call
-	ApiVersion int `json:"ApiVersion"`
-	// V2 Api Policies
-	HcnPolicyArgs []hcn.EndpointPolicy `json:"HcnPolicyArgs,omitempty"`
-	// V1 Api Policies
-	Policies []policy `json:"policies,omitempty"`
-	// Options to be passed in by the runtime
+	// ApiVersion specifies the policies type of HNS or HCN, select one of [1, 2].
+	// HNS is the v1 API, which is the default version and applies to dockershim.
+	// HCN is the v2 API, which can leverage HostComputeNamespace and use in containerd.
+	ApiVersion int `json:"apiVersion,omitempty"`
+	// Policies specifies the policy list for HNSEndpoint or HostComputeEndpoint.
+	Policies []Policy `json:"policies,omitempty"`
+	// RuntimeConfig represents the options to be passed in by the runtime.
 	RuntimeConfig RuntimeConfig `json:"runtimeConfig"`
-	// If true, adds a policy to endpoints to support loopback direct server return
-	LoopbackDSR bool `json:"loopbackDSR"`
+	// LoopbackDSR specifies whether to support loopback direct server return.
+	LoopbackDSR bool `json:"loopbackDSR,omitempty"`
 }
 
 type RuntimeDNS struct {
@@ -54,42 +55,67 @@ type PortMapEntry struct {
 	HostIP        string `json:"hostIP,omitempty"`
 }
 
+// constants of the supported Windows Socket protocol,
+// ref to https://docs.microsoft.com/en-us/dotnet/api/system.net.sockets.protocoltype.
+var protocolEnums = map[string]uint32{
+	"icmpv4": 1,
+	"igmp":   2,
+	"tcp":    6,
+	"udp":    17,
+	"icmpv6": 58,
+}
+
+func (p *PortMapEntry) GetProtocolEnum() (uint32, error) {
+	var u, err = strconv.ParseUint(p.Protocol, 0, 10)
+	if err != nil {
+		var pe, exist = protocolEnums[strings.ToLower(p.Protocol)]
+		if !exist {
+			return 0, errors.New("invalid protocol supplied to port mapping policy")
+		}
+		return pe, nil
+	}
+	return uint32(u), nil
+}
+
 type RuntimeConfig struct {
 	DNS      RuntimeDNS     `json:"dns"`
 	PortMaps []PortMapEntry `json:"portMappings,omitempty"`
 }
 
-type policy struct {
+type Policy struct {
 	Name  string          `json:"name"`
 	Value json.RawMessage `json:"value"`
 }
 
-func GetDefaultDestinationPrefix(ip *net.IP) string {
-	destinationPrefix := "0.0.0.0/0"
-	if ipv6 := ip.To4(); ipv6 == nil {
-		destinationPrefix = "::/0"
+// GetHNSEndpointPolicies converts the configuration policies to HNSEndpoint policies.
+func (n *NetConf) GetHNSEndpointPolicies() []json.RawMessage {
+	result := make([]json.RawMessage, 0, len(n.Policies))
+	for _, p := range n.Policies {
+		if !strings.EqualFold(p.Name, "EndpointPolicy") {
+			continue
+		}
+		result = append(result, p.Value)
 	}
-	return destinationPrefix
+	return result
 }
 
-func (n *NetConf) ApplyLoopbackDSR(ip *net.IP) {
-	value := fmt.Sprintf(`"Destinations" : ["%s"]`, ip.String())
-	if n.ApiVersion == 2 {
-		hcnLoopbackRoute := hcn.EndpointPolicy{
-			Type:     "OutBoundNAT",
-			Settings: []byte(fmt.Sprintf("{%s}", value)),
+// GetHostComputeEndpointPolicies converts the configuration policies to HostComputeEndpoint policies.
+func (n *NetConf) GetHostComputeEndpointPolicies() []hcn.EndpointPolicy {
+	result := make([]hcn.EndpointPolicy, 0, len(n.Policies))
+	for _, p := range n.Policies {
+		if !strings.EqualFold(p.Name, "EndpointPolicy") {
+			continue
 		}
-		n.HcnPolicyArgs = append(n.HcnPolicyArgs, hcnLoopbackRoute)
-	} else {
-		hnsLoopbackRoute := policy{
-			Name:  "EndpointPolicy",
-			Value: []byte(fmt.Sprintf(`{"Type": "OutBoundNAT", %s}`, value)),
+		var policy hcn.EndpointPolicy
+		if err := json.Unmarshal(p.Value, &policy); err != nil {
+			continue
 		}
-		n.Policies = append(n.Policies, hnsLoopbackRoute)
+		result = append(result, policy)
 	}
+	return result
 }
 
-// If runtime dns values are there use that else use cni conf supplied dns
+// GetDNS returns the DNS values if they are there use that else use netconf supplied DNS.
 func (n *NetConf) GetDNS() types.DNS {
 	dnsResult := n.DNS
 	if len(n.RuntimeConfig.DNS.Nameservers) > 0 {
@@ -101,136 +127,222 @@ func (n *NetConf) GetDNS() types.DNS {
 	return dnsResult
 }
 
-// MarshalPolicies converts the Endpoint policies in Policies
-// to HNS specific policies as Json raw bytes
-func (n *NetConf) MarshalPolicies() []json.RawMessage {
-	if n.Policies == nil {
-		n.Policies = make([]policy, 0)
+// ApplyLoopbackDSRPolicy configures the given IP to support loopback DSR.
+func (n *NetConf) ApplyLoopbackDSRPolicy(ip *net.IP) {
+	if err := hcn.DSRSupported(); err != nil || ip == nil {
+		return
 	}
 
-	result := make([]json.RawMessage, 0, len(n.Policies))
-	for _, p := range n.Policies {
+	toPolicyValue := func(addr string) json.RawMessage {
+		if n.ApiVersion == 2 {
+			return bprintf(`{"Type": "OutBoundNAT", "Settings": {"Destinations": ["%s"]}}`, addr)
+		}
+		return bprintf(`{"Type": "OutBoundNAT", "Destinations": ["%s"]}`, addr)
+	}
+	ipBytes := []byte(ip.String())
+
+	// find OutBoundNAT policy
+	for i := range n.Policies {
+		p := &n.Policies[i]
 		if !strings.EqualFold(p.Name, "EndpointPolicy") {
 			continue
 		}
 
-		result = append(result, p.Value)
-	}
-
-	return result
-}
-
-// ApplyOutboundNatPolicy applies NAT Policy in VFP using HNS
-// Simultaneously an exception is added for the network that has to be Nat'd
-func (n *NetConf) ApplyOutboundNatPolicy(nwToNat string) {
-	if n.Policies == nil {
-		n.Policies = make([]policy, 0)
-	}
-
-	nwToNatBytes := []byte(nwToNat)
-
-	for i, p := range n.Policies {
-		if !strings.EqualFold(p.Name, "EndpointPolicy") {
+		// filter OutBoundNAT policy
+		typeValue, _ := jsonparser.GetUnsafeString(p.Value, "Type")
+		if typeValue != "OutBoundNAT" {
 			continue
 		}
 
-		typeValue, err := jsonparser.GetUnsafeString(p.Value, "Type")
-		if err != nil || len(typeValue) == 0 {
+		// parse destination address list
+		var (
+			destinationsValue []byte
+			dt                jsonparser.ValueType
+		)
+		if n.ApiVersion == 2 {
+			destinationsValue, dt, _, _ = jsonparser.Get(p.Value, "Settings", "Destinations")
+		} else {
+			destinationsValue, dt, _, _ = jsonparser.Get(p.Value, "Destinations")
+		}
+
+		// skip if Destinations/DestinationList field is not found
+		if dt == jsonparser.NotExist {
 			continue
 		}
 
-		if !strings.EqualFold(typeValue, "OutBoundNAT") {
-			continue
-		}
-
-		exceptionListValue, dt, _, _ := jsonparser.Get(p.Value, "ExceptionList")
-		// OutBoundNAT must with ExceptionList, so don't need to judge jsonparser.NotExist
+		// return if found the given address
 		if dt == jsonparser.Array {
-			buf := bytes.Buffer{}
-			buf.WriteString(`{"Type": "OutBoundNAT", "ExceptionList": [`)
-
-			jsonparser.ArrayEach(exceptionListValue, func(value []byte, dataType jsonparser.ValueType, offset int, err error) {
+			var found bool
+			_, _ = jsonparser.ArrayEach(destinationsValue, func(value []byte, dataType jsonparser.ValueType, offset int, err error) {
 				if dataType == jsonparser.String && len(value) != 0 {
-					if bytes.Compare(value, nwToNatBytes) != 0 {
-						buf.WriteByte('"')
-						buf.Write(value)
-						buf.WriteByte('"')
-						buf.WriteByte(',')
+					if bytes.Compare(value, ipBytes) == 0 {
+						found = true
 					}
 				}
 			})
-
-			buf.WriteString(`"` + nwToNat + `"]}`)
-
-			n.Policies[i] = policy{
-				Name:  "EndpointPolicy",
-				Value: buf.Bytes(),
-			}
-		} else {
-			n.Policies[i] = policy{
-				Name:  "EndpointPolicy",
-				Value: []byte(`{"Type": "OutBoundNAT", "ExceptionList": ["` + nwToNat + `"]}`),
+			if found {
+				return
 			}
 		}
-
-		return
 	}
 
-	// didn't find the policyArg, add it
-	n.Policies = append(n.Policies, policy{
+	// or add a new OutBoundNAT if not found
+	n.Policies = append(n.Policies, Policy{
 		Name:  "EndpointPolicy",
-		Value: []byte(`{"Type": "OutBoundNAT", "ExceptionList": ["` + nwToNat + `"]}`),
+		Value: toPolicyValue(ip.String()),
 	})
 }
 
-// ApplyDefaultPAPolicy is used to configure a endpoint PA policy in HNS
-func (n *NetConf) ApplyDefaultPAPolicy(paAddress string) {
-	if n.Policies == nil {
-		n.Policies = make([]policy, 0)
+// ApplyOutboundNatPolicy applies the sNAT policy in HNS/HCN and configures the given CIDR as an exception.
+func (n *NetConf) ApplyOutboundNatPolicy(exceptionCIDR string) {
+	if exceptionCIDR == "" {
+		return
 	}
 
-	// if its already present, leave untouched
-	for i, p := range n.Policies {
+	toPolicyValue := func(cidr ...string) json.RawMessage {
+		if n.ApiVersion == 2 {
+			return bprintf(`{"Type": "OutBoundNAT", "Settings": {"Exceptions": ["%s"]}}`, strings.Join(cidr, `","`))
+		}
+		return bprintf(`{"Type": "OutBoundNAT", "ExceptionList": ["%s"]}`, strings.Join(cidr, `","`))
+	}
+	exceptionCIDRBytes := []byte(exceptionCIDR)
+
+	// find OutBoundNAT policy
+	for i := range n.Policies {
+		p := &n.Policies[i]
 		if !strings.EqualFold(p.Name, "EndpointPolicy") {
 			continue
 		}
 
-		paValue, dt, _, _ := jsonparser.Get(p.Value, "PA")
+		// filter OutBoundNAT policy
+		typeValue, _ := jsonparser.GetUnsafeString(p.Value, "Type")
+		if typeValue != "OutBoundNAT" {
+			continue
+		}
+
+		// parse exception CIDR list
+		var (
+			exceptionsValue []byte
+			dt              jsonparser.ValueType
+		)
+		if n.ApiVersion == 2 {
+			exceptionsValue, dt, _, _ = jsonparser.Get(p.Value, "Settings", "Exceptions")
+		} else {
+			exceptionsValue, dt, _, _ = jsonparser.Get(p.Value, "ExceptionList")
+		}
+
+		// skip if Exceptions/ExceptionList field is not found
 		if dt == jsonparser.NotExist {
 			continue
-		} else if dt == jsonparser.String && len(paValue) != 0 {
-			// found it, don't override
-			return
 		}
 
-		n.Policies[i] = policy{
-			Name:  "EndpointPolicy",
-			Value: []byte(`{"Type": "PA", "PA": "` + paAddress + `"}`),
+		// return if found the given CIDR
+		if dt == jsonparser.Array {
+			var found bool
+			_, _ = jsonparser.ArrayEach(exceptionsValue, func(value []byte, dataType jsonparser.ValueType, offset int, err error) {
+				if dataType == jsonparser.String && len(value) != 0 {
+					if bytes.Compare(value, exceptionCIDRBytes) == 0 {
+						found = true
+					}
+				}
+			})
+			if found {
+				return
+			}
 		}
-		return
 	}
 
-	// didn't find the policyArg, add it
-	n.Policies = append(n.Policies, policy{
+	// or add a new OutBoundNAT if not found
+	n.Policies = append(n.Policies, Policy{
 		Name:  "EndpointPolicy",
-		Value: []byte(`{"Type": "PA", "PA": "` + paAddress + `"}`),
+		Value: toPolicyValue(exceptionCIDR),
 	})
 }
 
-// ApplyPortMappingPolicy is used to configure HostPort<>ContainerPort mapping in HNS
-func (n *NetConf) ApplyPortMappingPolicy(portMappings []PortMapEntry) {
-	if portMappings == nil {
+// ApplyDefaultPAPolicy applies an endpoint PA policy in HNS/HCN.
+func (n *NetConf) ApplyDefaultPAPolicy(address string) {
+	if address == "" {
 		return
 	}
 
-	if n.Policies == nil {
-		n.Policies = make([]policy, 0)
+	toPolicyValue := func(addr string) json.RawMessage {
+		if n.ApiVersion == 2 {
+			return bprintf(`{"Type": "ProviderAddress", "Settings": {"ProviderAddress": "%s"}}`, addr)
+		}
+		return bprintf(`{"Type": "PA", "PA": "%s"}`, addr)
+	}
+	addressBytes := []byte(address)
+
+	// find ProviderAddress policy
+	for i := range n.Policies {
+		p := &n.Policies[i]
+		if !strings.EqualFold(p.Name, "EndpointPolicy") {
+			continue
+		}
+
+		// filter ProviderAddress policy
+		typeValue, _ := jsonparser.GetUnsafeString(p.Value, "Type")
+		if typeValue != "PA" && typeValue != "ProviderAddress" {
+			continue
+		}
+
+		// parse provider address
+		var (
+			paValue []byte
+			dt      jsonparser.ValueType
+		)
+		if n.ApiVersion == 2 {
+			paValue, dt, _, _ = jsonparser.Get(p.Value, "Settings", "ProviderAddress")
+		} else {
+			paValue, dt, _, _ = jsonparser.Get(p.Value, "PA")
+		}
+
+		// skip if ProviderAddress/PA field is not found
+		if dt == jsonparser.NotExist {
+			continue
+		}
+
+		// return if found the given address
+		if dt == jsonparser.String && bytes.Compare(paValue, addressBytes) == 0 {
+			return
+		}
 	}
 
-	for _, portMapping := range portMappings {
-		n.Policies = append(n.Policies, policy{
+	// or add a new ProviderAddress if not found
+	n.Policies = append(n.Policies, Policy{
+		Name:  "EndpointPolicy",
+		Value: toPolicyValue(address),
+	})
+}
+
+// ApplyPortMappingPolicy applies the host/container port mapping policies in HNS/HCN.
+func (n *NetConf) ApplyPortMappingPolicy(portMappings []PortMapEntry) {
+	if len(portMappings) == 0 {
+		return
+	}
+
+	toPolicyValue := func(p *PortMapEntry) json.RawMessage {
+		if n.ApiVersion == 2 {
+			var protocolEnum, _ = p.GetProtocolEnum()
+			return bprintf(`{"Type": "PortMapping", "Settings": {"InternalPort": %d, "ExternalPort": %d, "Protocol": %d, "VIP": "%s"}}`, p.ContainerPort, p.HostPort, protocolEnum, p.HostIP)
+		}
+		return bprintf(`{"Type": "NAT", "InternalPort": %d, "ExternalPort": %d, "Protocol": "%s"}`, p.ContainerPort, p.HostPort, p.Protocol)
+	}
+
+	for i := range portMappings {
+		p := &portMappings[i]
+		// skip the invalid protocol mapping
+		if _, err := p.GetProtocolEnum(); err != nil {
+			continue
+		}
+		n.Policies = append(n.Policies, Policy{
 			Name:  "EndpointPolicy",
-			Value: []byte(fmt.Sprintf(`{"Type": "NAT", "InternalPort": %d, "ExternalPort": %d, "Protocol": "%s"}`, portMapping.ContainerPort, portMapping.HostPort, portMapping.Protocol)),
+			Value: toPolicyValue(p),
 		})
 	}
 }
+
+// bprintf is similar to fmt.Sprintf and returns a byte array as result.
+func bprintf(format string, a ...interface{}) []byte {
+	return []byte(fmt.Sprintf(format, a...))
+}

pkg/hns/netconf_windows_test.go

@@ -15,221 +15,585 @@ package hns
 
 import (
 	"encoding/json"
+	"net"
 
+	"github.com/Microsoft/hcsshim/hcn"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 )
 
-var _ = Describe("HNS NetConf", func() {
-	Describe("ApplyOutBoundNATPolicy", func() {
-		Context("when not set by user", func() {
-			It("sets it by adding a policy", func() {
+var _ = Describe("NetConf", func() {
+	Describe("ApplyLoopbackDSRPolicy", func() {
+		Context("via v1 api", func() {
+			var n NetConf
+			BeforeEach(func() {
+				n = NetConf{}
+			})
 
-				// apply it
-				n := NetConf{}
-				n.ApplyOutboundNatPolicy("192.168.0.0/16")
+			It("filter out duplicated IP", func() {
+				// mock duplicated IP
+				ip := net.ParseIP("172.16.0.12")
+				n.ApplyLoopbackDSRPolicy(&ip)
+				n.ApplyLoopbackDSRPolicy(&ip)
 
+				// only one policy
 				addlArgs := n.Policies
 				Expect(addlArgs).Should(HaveLen(1))
 
+				// normal type judgement
 				policy := addlArgs[0]
 				Expect(policy.Name).Should(Equal("EndpointPolicy"))
-
 				value := make(map[string]interface{})
 				json.Unmarshal(policy.Value, &value)
-
 				Expect(value).Should(HaveKey("Type"))
-				Expect(value).Should(HaveKey("ExceptionList"))
 				Expect(value["Type"]).Should(Equal("OutBoundNAT"))
+				Expect(value).Should(HaveKey("Destinations"))
 
-				exceptionList := value["ExceptionList"].([]interface{})
-				Expect(exceptionList).Should(HaveLen(1))
-				Expect(exceptionList[0].(string)).Should(Equal("192.168.0.0/16"))
+				// and only one item
+				destinationList := value["Destinations"].([]interface{})
+				Expect(destinationList).Should(HaveLen(1))
+				Expect(destinationList[0].(string)).Should(Equal("172.16.0.12"))
+			})
+
+			It("append different IP", func() {
+				// mock different IP
+				ip1 := net.ParseIP("172.16.0.12")
+				n.ApplyLoopbackDSRPolicy(&ip1)
+				ip2 := net.ParseIP("172.16.0.13")
+				n.ApplyLoopbackDSRPolicy(&ip2)
+
+				// will be two policies
+				addlArgs := n.Policies
+				Expect(addlArgs).Should(HaveLen(2))
+
+				// normal type judgement
+				policy := addlArgs[1] // pick second item
+				Expect(policy.Name).Should(Equal("EndpointPolicy"))
+				value := make(map[string]interface{})
+				json.Unmarshal(policy.Value, &value)
+				Expect(value).Should(HaveKey("Type"))
+				Expect(value["Type"]).Should(Equal("OutBoundNAT"))
+				Expect(value).Should(HaveKey("Destinations"))
+
+				// only one item
+				destinationList := value["Destinations"].([]interface{})
+				Expect(destinationList).Should(HaveLen(1))
+				Expect(destinationList[0].(string)).Should(Equal("172.16.0.13"))
 			})
 		})
 
-		Context("when set by user", func() {
-			It("appends exceptions to the existing policy", func() {
-				// first set it
-				n := NetConf{}
-				n.ApplyOutboundNatPolicy("192.168.0.0/16")
+		Context("via v2 api", func() {
+			var n NetConf
+			BeforeEach(func() {
+				n = NetConf{ApiVersion: 2}
+			})
 
-				// then attempt to update it
-				n.ApplyOutboundNatPolicy("10.244.0.0/16")
+			It("filter out duplicated IP", func() {
+				// mock duplicated IP
+				ip := net.ParseIP("172.16.0.12")
+				n.ApplyLoopbackDSRPolicy(&ip)
+				n.ApplyLoopbackDSRPolicy(&ip)
 
-				// it should be unchanged!
+				// only one policy
 				addlArgs := n.Policies
 				Expect(addlArgs).Should(HaveLen(1))
 
+				// normal type judgement
 				policy := addlArgs[0]
 				Expect(policy.Name).Should(Equal("EndpointPolicy"))
-
-				var value map[string]interface{}
+				value := make(map[string]interface{})
 				json.Unmarshal(policy.Value, &value)
-
 				Expect(value).Should(HaveKey("Type"))
-				Expect(value).Should(HaveKey("ExceptionList"))
 				Expect(value["Type"]).Should(Equal("OutBoundNAT"))
+				Expect(value).Should(HaveKey("Settings"))
 
+				// and only one item
+				settings := value["Settings"].(map[string]interface{})
+				destinationList := settings["Destinations"].([]interface{})
+				Expect(destinationList).Should(HaveLen(1))
+				Expect(destinationList[0].(string)).Should(Equal("172.16.0.12"))
+			})
+
+			It("append different IP", func() {
+				// mock different IP
+				ip1 := net.ParseIP("172.16.0.12")
+				n.ApplyLoopbackDSRPolicy(&ip1)
+				ip2 := net.ParseIP("172.16.0.13")
+				n.ApplyLoopbackDSRPolicy(&ip2)
+
+				// will be two policies
+				addlArgs := n.Policies
+				Expect(addlArgs).Should(HaveLen(2))
+
+				// normal type judgement
+				policy := addlArgs[1] // pick second item
+				Expect(policy.Name).Should(Equal("EndpointPolicy"))
+				value := make(map[string]interface{})
+				json.Unmarshal(policy.Value, &value)
+				Expect(value).Should(HaveKey("Type"))
+				Expect(value["Type"]).Should(Equal("OutBoundNAT"))
+				Expect(value).Should(HaveKey("Settings"))
+
+				// only one item
+				settings := value["Settings"].(map[string]interface{})
+				destinationList := settings["Destinations"].([]interface{})
+				Expect(destinationList).Should(HaveLen(1))
+				Expect(destinationList[0].(string)).Should(Equal("172.16.0.13"))
+			})
+		})
+	})
+
+	Describe("ApplyOutBoundNATPolicy", func() {
+		Context("via v1 api", func() {
+			var n NetConf
+			BeforeEach(func() {
+				n = NetConf{}
+			})
+
+			It("append different IP", func() {
+				// mock different IP
+				n.ApplyOutboundNatPolicy("192.168.0.0/16")
+				n.ApplyOutboundNatPolicy("10.244.0.0/16")
+
+				// will be two policies
+				addlArgs := n.Policies
+				Expect(addlArgs).Should(HaveLen(2))
+
+				// normal type judgement
+				policy := addlArgs[1] // pick second item
+				Expect(policy.Name).Should(Equal("EndpointPolicy"))
+				value := make(map[string]interface{})
+				json.Unmarshal(policy.Value, &value)
+				Expect(value).Should(HaveKey("Type"))
+				Expect(value["Type"]).Should(Equal("OutBoundNAT"))
+				Expect(value).Should(HaveKey("ExceptionList"))
+
+				// but get two items
 				exceptionList := value["ExceptionList"].([]interface{})
-				Expect(exceptionList).Should(HaveLen(2))
-				Expect(exceptionList[0].(string)).Should(Equal("192.168.0.0/16"))
-				Expect(exceptionList[1].(string)).Should(Equal("10.244.0.0/16"))
+				Expect(exceptionList).Should(HaveLen(1))
+				Expect(exceptionList[0].(string)).Should(Equal("10.244.0.0/16"))
+			})
+
+			It("append a new one if there is not an exception OutBoundNAT policy", func() {
+				// mock different OutBoundNAT routes
+				n.Policies = []Policy{
+					{
+						Name:  "EndpointPolicy",
+						Value: bprintf(`{"Type": "OutBoundNAT", "OtherList": []}`),
+					},
+				}
+				n.ApplyOutboundNatPolicy("10.244.0.0/16")
+
+				// has two policies
+				addlArgs := n.Policies
+				Expect(addlArgs).Should(HaveLen(2))
+
+				// normal type judgement
+				policy := addlArgs[0]
+				Expect(policy.Name).Should(Equal("EndpointPolicy"))
+				value := make(map[string]interface{})
+				json.Unmarshal(policy.Value, &value)
+				Expect(value).Should(HaveKey("Type"))
+				Expect(value["Type"]).Should(Equal("OutBoundNAT"))
+				Expect(value).Should(HaveKey("OtherList"))
+				policy = addlArgs[1]
+				value = make(map[string]interface{})
+				json.Unmarshal(policy.Value, &value)
+				Expect(value).Should(HaveKey("Type"))
+				Expect(value["Type"]).Should(Equal("OutBoundNAT"))
+				Expect(value).Should(HaveKey("ExceptionList"))
+
+				// only get one item
+				exceptionList := value["ExceptionList"].([]interface{})
+				Expect(exceptionList).Should(HaveLen(1))
+				Expect(exceptionList[0].(string)).Should(Equal("10.244.0.0/16"))
+			})
+
+			It("nothing to do if CIDR is blank", func() {
+				// mock different OutBoundNAT routes
+				n.Policies = []Policy{
+					{
+						Name:  "EndpointPolicy",
+						Value: bprintf(`{"Type": "OutBoundNAT", "ExceptionList": []}`),
+					},
+				}
+				n.ApplyOutboundNatPolicy("")
+
+				// only one policy
+				addlArgs := n.Policies
+				Expect(addlArgs).Should(HaveLen(1))
+
+				// normal type judgement
+				policy := addlArgs[0]
+				Expect(policy.Name).Should(Equal("EndpointPolicy"))
+				value := make(map[string]interface{})
+				json.Unmarshal(policy.Value, &value)
+				Expect(value).Should(HaveKey("Type"))
+				Expect(value["Type"]).Should(Equal("OutBoundNAT"))
+				Expect(value).Should(HaveKey("ExceptionList"))
+
+				// empty list
+				Expect(value["ExceptionList"]).ShouldNot(BeNil())
+				Expect(value["ExceptionList"]).Should(HaveLen(0))
+			})
+		})
+
+		Context("via v2 api", func() {
+			var n NetConf
+			BeforeEach(func() {
+				n = NetConf{ApiVersion: 2}
+			})
+
+			It("append different IP", func() {
+				// mock different IP
+				n.ApplyOutboundNatPolicy("192.168.0.0/16")
+				n.ApplyOutboundNatPolicy("10.244.0.0/16")
+
+				// will be two policies
+				addlArgs := n.Policies
+				Expect(addlArgs).Should(HaveLen(2))
+
+				// normal type judgement
+				policy := addlArgs[1] // pick second item
+				Expect(policy.Name).Should(Equal("EndpointPolicy"))
+				value := make(map[string]interface{})
+				json.Unmarshal(policy.Value, &value)
+				Expect(value).Should(HaveKey("Type"))
+				Expect(value["Type"]).Should(Equal("OutBoundNAT"))
+				Expect(value).Should(HaveKey("Settings"))
+
+				// but get two items
+				settings := value["Settings"].(map[string]interface{})
+				exceptionList := settings["Exceptions"].([]interface{})
+				Expect(exceptionList).Should(HaveLen(1))
+				Expect(exceptionList[0].(string)).Should(Equal("10.244.0.0/16"))
+			})
+
+			It("append a new one if there is not an exception OutBoundNAT policy", func() {
+				// mock different OutBoundNAT routes
+				n.Policies = []Policy{
+					{
+						Name:  "EndpointPolicy",
+						Value: bprintf(`{"Type": "OutBoundNAT", "Settings": {"Others": []}}`),
+					},
+				}
+				n.ApplyOutboundNatPolicy("10.244.0.0/16")
+
+				// has two policies
+				addlArgs := n.Policies
+				Expect(addlArgs).Should(HaveLen(2))
+
+				// normal type judgement
+				policy := addlArgs[0]
+				Expect(policy.Name).Should(Equal("EndpointPolicy"))
+				value := make(map[string]interface{})
+				json.Unmarshal(policy.Value, &value)
+				Expect(value).Should(HaveKey("Type"))
+				Expect(value["Type"]).Should(Equal("OutBoundNAT"))
+				Expect(value).Should(HaveKey("Settings"))
+				Expect(value["Settings"]).Should(HaveKey("Others"))
+				policy = addlArgs[1]
+				value = make(map[string]interface{})
+				json.Unmarshal(policy.Value, &value)
+				Expect(value).Should(HaveKey("Type"))
+				Expect(value["Type"]).Should(Equal("OutBoundNAT"))
+				Expect(value).Should(HaveKey("Settings"))
+
+				// only get one item
+				settings := value["Settings"].(map[string]interface{})
+				exceptionList := settings["Exceptions"].([]interface{})
+				Expect(exceptionList).Should(HaveLen(1))
+				Expect(exceptionList[0].(string)).Should(Equal("10.244.0.0/16"))
+			})
+
+			It("nothing to do if CIDR is blank", func() {
+				// mock different OutBoundNAT routes
+				n.Policies = []Policy{
+					{
+						Name:  "EndpointPolicy",
+						Value: bprintf(`{"Type": "OutBoundNAT", "Settings": {"Exceptions": []}}`),
+					},
+				}
+				n.ApplyOutboundNatPolicy("")
+
+				// only one policy
+				addlArgs := n.Policies
+				Expect(addlArgs).Should(HaveLen(1))
+
+				// normal type judgement
+				policy := addlArgs[0]
+				Expect(policy.Name).Should(Equal("EndpointPolicy"))
+				value := make(map[string]interface{})
+				json.Unmarshal(policy.Value, &value)
+				Expect(value).Should(HaveKey("Type"))
+				Expect(value["Type"]).Should(Equal("OutBoundNAT"))
+				Expect(value).Should(HaveKey("Settings"))
+
+				// empty list
+				settings := value["Settings"].(map[string]interface{})
+				Expect(settings["Exceptions"]).ShouldNot(BeNil())
+				Expect(settings["Exceptions"]).Should(HaveLen(0))
 			})
 		})
 	})
 
 	Describe("ApplyDefaultPAPolicy", func() {
-		Context("when not set by user", func() {
-			It("sets it by adding a policy", func() {
-
-				n := NetConf{}
-				n.ApplyDefaultPAPolicy("192.168.0.1")
-
-				addlArgs := n.Policies
-				Expect(addlArgs).Should(HaveLen(1))
-
-				policy := addlArgs[0]
-				Expect(policy.Name).Should(Equal("EndpointPolicy"))
-
-				value := make(map[string]interface{})
-				json.Unmarshal(policy.Value, &value)
-
-				Expect(value).Should(HaveKey("Type"))
-				Expect(value["Type"]).Should(Equal("PA"))
-
-				paAddress := value["PA"].(string)
-				Expect(paAddress).Should(Equal("192.168.0.1"))
+		Context("via v1 api", func() {
+			var n NetConf
+			BeforeEach(func() {
+				n = NetConf{}
 			})
-		})
 
-		Context("when set by user", func() {
-			It("does not override", func() {
-				n := NetConf{}
+			It("append different IP", func() {
+				// mock different IP
 				n.ApplyDefaultPAPolicy("192.168.0.1")
 				n.ApplyDefaultPAPolicy("192.168.0.2")
 
+				// will be two policies
 				addlArgs := n.Policies
-				Expect(addlArgs).Should(HaveLen(1))
+				Expect(addlArgs).Should(HaveLen(2))
 
-				policy := addlArgs[0]
+				// normal type judgement
+				policy := addlArgs[1] // judge second item
 				Expect(policy.Name).Should(Equal("EndpointPolicy"))
-
 				value := make(map[string]interface{})
 				json.Unmarshal(policy.Value, &value)
-
 				Expect(value).Should(HaveKey("Type"))
 				Expect(value["Type"]).Should(Equal("PA"))
 
+				// compare with second item
 				paAddress := value["PA"].(string)
-				Expect(paAddress).Should(Equal("192.168.0.1"))
-				Expect(paAddress).ShouldNot(Equal("192.168.0.2"))
+				Expect(paAddress).Should(Equal("192.168.0.2"))
+			})
+
+			It("nothing to do if IP is blank", func() {
+				// mock different policy
+				n.Policies = []Policy{
+					{
+						Name:  "EndpointPolicy",
+						Value: bprintf(`{"Type": "OutBoundNAT", "Exceptions": ["192.168.0.0/16"]}`),
+					},
+				}
+				n.ApplyDefaultPAPolicy("")
+
+				// nothing
+				addlArgs := n.Policies
+				Expect(addlArgs).Should(HaveLen(1))
+			})
+		})
+
+		Context("via v2 api", func() {
+			var n NetConf
+			BeforeEach(func() {
+				n = NetConf{ApiVersion: 2}
+			})
+
+			It("append different IP", func() {
+				// mock different IP
+				n.ApplyDefaultPAPolicy("192.168.0.1")
+				n.ApplyDefaultPAPolicy("192.168.0.2")
+
+				// will be two policies
+				addlArgs := n.Policies
+				Expect(addlArgs).Should(HaveLen(2))
+
+				// normal type judgement
+				policy := addlArgs[1] // judge second item
+				Expect(policy.Name).Should(Equal("EndpointPolicy"))
+				value := make(map[string]interface{})
+				json.Unmarshal(policy.Value, &value)
+				Expect(value).Should(HaveKey("Type"))
+				Expect(value["Type"]).Should(Equal("ProviderAddress"))
+				Expect(value).Should(HaveKey("Settings"))
+
+				// compare with second item
+				settings := value["Settings"].(map[string]interface{})
+				paAddress := settings["ProviderAddress"].(string)
+				Expect(paAddress).Should(Equal("192.168.0.2"))
+			})
+
+			It("nothing to do if IP is blank", func() {
+				// mock different policy
+				n.Policies = []Policy{
+					{
+						Name:  "EndpointPolicy",
+						Value: bprintf(`{"Type": "OutBoundNAT", "Settings": {"Exceptions": ["192.168.0.0/16"]}}`),
+					},
+				}
+				n.ApplyDefaultPAPolicy("")
+
+				// nothing
+				addlArgs := n.Policies
+				Expect(addlArgs).Should(HaveLen(1))
 			})
 		})
 	})
 
 	Describe("ApplyPortMappingPolicy", func() {
-		Context("when portMappings not activated", func() {
-			It("does nothing", func() {
-				n := NetConf{}
+		Context("via v1 api", func() {
+			var n NetConf
+			BeforeEach(func() {
+				n = NetConf{}
+			})
+
+			It("nothing to do if input is empty", func() {
 				n.ApplyPortMappingPolicy(nil)
 				Expect(n.Policies).Should(BeNil())
 
 				n.ApplyPortMappingPolicy([]PortMapEntry{})
-				Expect(n.Policies).Should(HaveLen(0))
+				Expect(n.Policies).Should(BeNil())
 			})
-		})
 
-		Context("when portMappings is activated", func() {
-			It("creates NAT policies", func() {
-				n := NetConf{}
+			It("create one NAT policy", func() {
+				// mock different IP
 				n.ApplyPortMappingPolicy([]PortMapEntry{
 					{
 						ContainerPort: 80,
 						HostPort:      8080,
 						Protocol:      "TCP",
-						HostIP:        "ignored",
+						HostIP:        "192.168.1.2",
 					},
 				})
 
-				Expect(n.Policies).Should(HaveLen(1))
+				// only one item
+				addlArgs := n.Policies
+				Expect(addlArgs).Should(HaveLen(1))
 
-				policy := n.Policies[0]
+				// normal type judgement
+				policy := addlArgs[0]
 				Expect(policy.Name).Should(Equal("EndpointPolicy"))
-
 				value := make(map[string]interface{})
 				json.Unmarshal(policy.Value, &value)
-
 				Expect(value).Should(HaveKey("Type"))
 				Expect(value["Type"]).Should(Equal("NAT"))
 
+				// compare all values
 				Expect(value).Should(HaveKey("InternalPort"))
 				Expect(value["InternalPort"]).Should(Equal(float64(80)))
-
 				Expect(value).Should(HaveKey("ExternalPort"))
 				Expect(value["ExternalPort"]).Should(Equal(float64(8080)))
-
 				Expect(value).Should(HaveKey("Protocol"))
 				Expect(value["Protocol"]).Should(Equal("TCP"))
 			})
 		})
-	})
 
-	Describe("MarshalPolicies", func() {
-		Context("when not set by user", func() {
-			It("sets it by adding a policy", func() {
-
-				n := NetConf{
-					Policies: []policy{
-						{
-							Name:  "EndpointPolicy",
-							Value: []byte(`{"someKey": "someValue"}`),
-						},
-						{
-							Name:  "someOtherType",
-							Value: []byte(`{"someOtherKey": "someOtherValue"}`),
-						},
-					},
-				}
-
-				result := n.MarshalPolicies()
-				Expect(len(result)).To(Equal(1))
-
-				policy := make(map[string]interface{})
-				err := json.Unmarshal(result[0], &policy)
-				Expect(err).ToNot(HaveOccurred())
-				Expect(policy).Should(HaveKey("someKey"))
-				Expect(policy["someKey"]).To(Equal("someValue"))
+		Context("via v2 api", func() {
+			var n NetConf
+			BeforeEach(func() {
+				n = NetConf{ApiVersion: 2}
 			})
-		})
 
-		Context("when set by user", func() {
-			It("appends exceptions to the existing policy", func() {
-				// first set it
-				n := NetConf{}
-				n.ApplyOutboundNatPolicy("192.168.0.0/16")
+			It("nothing to do if input is empty", func() {
+				n.ApplyPortMappingPolicy(nil)
+				Expect(n.Policies).Should(BeNil())
 
-				// then attempt to update it
-				n.ApplyOutboundNatPolicy("10.244.0.0/16")
+				n.ApplyPortMappingPolicy([]PortMapEntry{})
+				Expect(n.Policies).Should(BeNil())
+			})
 
-				// it should be unchanged!
+			It("creates one NAT policy", func() {
+				// mock different IP
+				n.ApplyPortMappingPolicy([]PortMapEntry{
+					{
+						ContainerPort: 80,
+						HostPort:      8080,
+						Protocol:      "TCP",
+						HostIP:        "192.168.1.2",
+					},
+				})
+
+				// only one item
 				addlArgs := n.Policies
 				Expect(addlArgs).Should(HaveLen(1))
 
+				// normal type judgement
 				policy := addlArgs[0]
 				Expect(policy.Name).Should(Equal("EndpointPolicy"))
-
-				var value map[string]interface{}
+				value := make(map[string]interface{})
 				json.Unmarshal(policy.Value, &value)
-
 				Expect(value).Should(HaveKey("Type"))
-				Expect(value).Should(HaveKey("ExceptionList"))
-				Expect(value["Type"]).Should(Equal("OutBoundNAT"))
+				Expect(value["Type"]).Should(Equal("PortMapping"))
+				Expect(value).Should(HaveKey("Settings"))
 
-				exceptionList := value["ExceptionList"].([]interface{})
-				Expect(exceptionList).Should(HaveLen(2))
-				Expect(exceptionList[0].(string)).Should(Equal("192.168.0.0/16"))
-				Expect(exceptionList[1].(string)).Should(Equal("10.244.0.0/16"))
+				// compare all values
+				settings := value["Settings"].(map[string]interface{})
+				Expect(settings).Should(HaveKey("InternalPort"))
+				Expect(settings["InternalPort"]).Should(Equal(float64(80)))
+				Expect(settings).Should(HaveKey("ExternalPort"))
+				Expect(settings["ExternalPort"]).Should(Equal(float64(8080)))
+				Expect(settings).Should(HaveKey("Protocol"))
+				Expect(settings["Protocol"]).Should(Equal(float64(6)))
+				Expect(settings).Should(HaveKey("VIP"))
+				Expect(settings["VIP"]).Should(Equal("192.168.1.2"))
+			})
+		})
+	})
+
+	Describe("GetXEndpointPolicies", func() {
+		Context("via v1 api", func() {
+			var n NetConf
+			BeforeEach(func() {
+				n = NetConf{}
+			})
+
+			It("GetHNSEndpointPolicies", func() {
+				// mock different policies
+				n.Policies = []Policy{
+					{
+						Name:  "EndpointPolicy",
+						Value: []byte(`{"Type": "OutBoundNAT", "ExceptionList": [ "192.168.1.2" ]}`),
+					},
+					{
+						Name:  "someOtherType",
+						Value: []byte(`{"someOtherKey": "someOtherValue"}`),
+					},
+				}
+
+				// only one valid item
+				result := n.GetHNSEndpointPolicies()
+				Expect(len(result)).To(Equal(1))
+
+				// normal type judgement
+				policy := make(map[string]interface{})
+				err := json.Unmarshal(result[0], &policy)
+				Expect(err).ToNot(HaveOccurred())
+				Expect(policy).Should(HaveKey("Type"))
+				Expect(policy["Type"]).To(Equal("OutBoundNAT"))
+				Expect(policy).Should(HaveKey("ExceptionList"))
+				Expect(policy["ExceptionList"]).To(ContainElement("192.168.1.2"))
+			})
+		})
+
+		Context("via v2 api", func() {
+			var n NetConf
+			BeforeEach(func() {
+				n = NetConf{ApiVersion: 2}
+			})
+
+			It("GetHostComputeEndpointPolicies", func() {
+				// mock different policies
+				n.Policies = []Policy{
+					{
+						Name:  "EndpointPolicy",
+						Value: []byte(`{"Type": "OutBoundNAT", "Settings": {"Exceptions": [ "192.168.1.2" ]}}`),
+					},
+					{
+						Name:  "someOtherType",
+						Value: []byte(`{"someOtherKey": "someOtherValue"}`),
+					},
+				}
+
+				// only one valid item
+				result := n.GetHostComputeEndpointPolicies()
+				Expect(len(result)).To(Equal(1))
+
+				// normal type judgement
+				policy := result[0]
+				Expect(policy.Type).Should(Equal(hcn.OutBoundNAT))
+				settings := make(map[string]interface{})
+				err := json.Unmarshal(policy.Settings, &settings)
+				Expect(err).ToNot(HaveOccurred())
+				Expect(settings["Exceptions"]).To(ContainElement("192.168.1.2"))
 			})
 		})
 	})

pkg/ip/cidr.go

@@ -19,43 +19,87 @@ import (
 	"net"
 )
 
-// NextIP returns IP incremented by 1
+// NextIP returns IP incremented by 1, if IP is invalid, return nil
 func NextIP(ip net.IP) net.IP {
-	i := ipToInt(ip)
-	return intToIP(i.Add(i, big.NewInt(1)))
+	normalizedIP := normalizeIP(ip)
+	if normalizedIP == nil {
+		return nil
+	}
+
+	i := ipToInt(normalizedIP)
+	return intToIP(i.Add(i, big.NewInt(1)), len(normalizedIP) == net.IPv6len)
 }
 
-// PrevIP returns IP decremented by 1
+// PrevIP returns IP decremented by 1, if IP is invalid, return nil
 func PrevIP(ip net.IP) net.IP {
-	i := ipToInt(ip)
-	return intToIP(i.Sub(i, big.NewInt(1)))
+	normalizedIP := normalizeIP(ip)
+	if normalizedIP == nil {
+		return nil
+	}
+
+	i := ipToInt(normalizedIP)
+	return intToIP(i.Sub(i, big.NewInt(1)), len(normalizedIP) == net.IPv6len)
 }
 
 // Cmp compares two IPs, returning the usual ordering:
 // a < b : -1
 // a == b : 0
 // a > b : 1
+// incomparable : -2
 func Cmp(a, b net.IP) int {
-	aa := ipToInt(a)
-	bb := ipToInt(b)
-	return aa.Cmp(bb)
+	normalizedA := normalizeIP(a)
+	normalizedB := normalizeIP(b)
+
+	if len(normalizedA) == len(normalizedB) && len(normalizedA) != 0 {
+		return ipToInt(normalizedA).Cmp(ipToInt(normalizedB))
+	}
+
+	return -2
 }
 
 func ipToInt(ip net.IP) *big.Int {
-	if v := ip.To4(); v != nil {
-		return big.NewInt(0).SetBytes(v)
+	return big.NewInt(0).SetBytes(ip)
+}
+
+func intToIP(i *big.Int, isIPv6 bool) net.IP {
+	intBytes := i.Bytes()
+
+	if len(intBytes) == net.IPv4len || len(intBytes) == net.IPv6len {
+		return intBytes
 	}
-	return big.NewInt(0).SetBytes(ip.To16())
+
+	if isIPv6 {
+		return append(make([]byte, net.IPv6len-len(intBytes)), intBytes...)
+	}
+
+	return append(make([]byte, net.IPv4len-len(intBytes)), intBytes...)
 }
 
-func intToIP(i *big.Int) net.IP {
-	return net.IP(i.Bytes())
+// normalizeIP will normalize IP by family,
+// IPv4 : 4-byte form
+// IPv6 : 16-byte form
+// others : nil
+func normalizeIP(ip net.IP) net.IP {
+	if ipTo4 := ip.To4(); ipTo4 != nil {
+		return ipTo4
+	}
+	return ip.To16()
 }
 
-// Network masks off the host portion of the IP
+// Network masks off the host portion of the IP, if IPNet is invalid,
+// return nil
 func Network(ipn *net.IPNet) *net.IPNet {
+	if ipn == nil {
+		return nil
+	}
+
+	maskedIP := ipn.IP.Mask(ipn.Mask)
+	if maskedIP == nil {
+		return nil
+	}
+
 	return &net.IPNet{
-		IP:   ipn.IP.Mask(ipn.Mask),
+		IP:   maskedIP,
 		Mask: ipn.Mask,
 	}
 }

pkg/ip/cidr_test.go

@@ -0,0 +1,247 @@
+// Copyright 2022 CNI authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package ip
+
+import (
+	"net"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("CIDR functions", func() {
+	It("NextIP", func() {
+		testCases := []struct {
+			ip     net.IP
+			nextIP net.IP
+		}{
+			{
+				[]byte{192, 0, 2},
+				nil,
+			},
+			{
+				net.ParseIP("192.168.0.1"),
+				net.IPv4(192, 168, 0, 2).To4(),
+			},
+			{
+				net.ParseIP("192.168.0.255"),
+				net.IPv4(192, 168, 1, 0).To4(),
+			},
+			{
+				net.ParseIP("0.1.0.5"),
+				net.IPv4(0, 1, 0, 6).To4(),
+			},
+			{
+				net.ParseIP("AB12::123"),
+				net.ParseIP("AB12::124"),
+			},
+			{
+				net.ParseIP("AB12::FFFF"),
+				net.ParseIP("AB12::1:0"),
+			},
+			{
+				net.ParseIP("0::123"),
+				net.ParseIP("0::124"),
+			},
+		}
+
+		for _, test := range testCases {
+			ip := NextIP(test.ip)
+
+			Expect(ip).To(Equal(test.nextIP))
+		}
+	})
+
+	It("PrevIP", func() {
+		testCases := []struct {
+			ip     net.IP
+			prevIP net.IP
+		}{
+			{
+				[]byte{192, 0, 2},
+				nil,
+			},
+			{
+				net.ParseIP("192.168.0.2"),
+				net.IPv4(192, 168, 0, 1).To4(),
+			},
+			{
+				net.ParseIP("192.168.1.0"),
+				net.IPv4(192, 168, 0, 255).To4(),
+			},
+			{
+				net.ParseIP("0.1.0.5"),
+				net.IPv4(0, 1, 0, 4).To4(),
+			},
+			{
+				net.ParseIP("AB12::123"),
+				net.ParseIP("AB12::122"),
+			},
+			{
+				net.ParseIP("AB12::1:0"),
+				net.ParseIP("AB12::FFFF"),
+			},
+			{
+				net.ParseIP("0::124"),
+				net.ParseIP("0::123"),
+			},
+		}
+
+		for _, test := range testCases {
+			ip := PrevIP(test.ip)
+
+			Expect(ip).To(Equal(test.prevIP))
+		}
+	})
+
+	It("Cmp", func() {
+		testCases := []struct {
+			a      net.IP
+			b      net.IP
+			result int
+		}{
+			{
+				net.ParseIP("192.168.0.2"),
+				nil,
+				-2,
+			},
+			{
+				net.ParseIP("192.168.0.2"),
+				[]byte{192, 168, 5},
+				-2,
+			},
+			{
+				net.ParseIP("192.168.0.2"),
+				net.ParseIP("AB12::123"),
+				-2,
+			},
+			{
+				net.ParseIP("192.168.0.2"),
+				net.ParseIP("192.168.0.5"),
+				-1,
+			},
+			{
+				net.ParseIP("192.168.0.2"),
+				net.ParseIP("192.168.0.5").To4(),
+				-1,
+			},
+			{
+				net.ParseIP("192.168.0.10"),
+				net.ParseIP("192.168.0.5"),
+				1,
+			},
+			{
+				net.ParseIP("192.168.0.10"),
+				net.ParseIP("192.168.0.10"),
+				0,
+			},
+			{
+				net.ParseIP("192.168.0.10"),
+				net.ParseIP("192.168.0.10").To4(),
+				0,
+			},
+			{
+				net.ParseIP("AB12::122"),
+				net.ParseIP("AB12::123"),
+				-1,
+			},
+			{
+				net.ParseIP("AB12::210"),
+				net.ParseIP("AB12::123"),
+				1,
+			},
+			{
+				net.ParseIP("AB12::210"),
+				net.ParseIP("AB12::210"),
+				0,
+			},
+		}
+
+		for _, test := range testCases {
+			result := Cmp(test.a, test.b)
+
+			Expect(result).To(Equal(test.result))
+		}
+	})
+
+	It("Network", func() {
+		testCases := []struct {
+			ipNet  *net.IPNet
+			result *net.IPNet
+		}{
+			{
+				nil,
+				nil,
+			},
+			{
+				&net.IPNet{
+					IP:   nil,
+					Mask: net.IPv4Mask(255, 255, 255, 0),
+				},
+				nil,
+			},
+			{
+				&net.IPNet{
+					IP:   net.IPv4(192, 168, 0, 1),
+					Mask: nil,
+				},
+				nil,
+			},
+			{
+				&net.IPNet{
+					IP:   net.ParseIP("AB12::123"),
+					Mask: net.IPv4Mask(255, 255, 255, 0),
+				},
+				nil,
+			},
+			{
+				&net.IPNet{
+					IP:   net.IPv4(192, 168, 0, 100).To4(),
+					Mask: net.CIDRMask(120, 128),
+				},
+				&net.IPNet{
+					IP:   net.IPv4(192, 168, 0, 0).To4(),
+					Mask: net.CIDRMask(120, 128),
+				},
+			},
+			{
+				&net.IPNet{
+					IP:   net.IPv4(192, 168, 0, 100),
+					Mask: net.CIDRMask(24, 32),
+				},
+				&net.IPNet{
+					IP:   net.IPv4(192, 168, 0, 0).To4(),
+					Mask: net.CIDRMask(24, 32),
+				},
+			},
+			{
+				&net.IPNet{
+					IP:   net.ParseIP("AB12::123"),
+					Mask: net.CIDRMask(120, 128),
+				},
+				&net.IPNet{
+					IP:   net.ParseIP("AB12::100"),
+					Mask: net.CIDRMask(120, 128),
+				},
+			},
+		}
+
+		for _, test := range testCases {
+			result := Network(test.ipNet)
+
+			Expect(result).To(Equal(test.result))
+		}
+	})
+})

pkg/ip/ip.go

@@ -0,0 +1,105 @@
+// Copyright 2021 CNI authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package ip
+
+import (
+	"fmt"
+	"net"
+	"strings"
+)
+
+// IP is a CNI maintained type inherited from net.IPNet which can
+// represent a single IP address with or without prefix.
+type IP struct {
+	net.IPNet
+}
+
+// newIP will create an IP with net.IP and net.IPMask
+func newIP(ip net.IP, mask net.IPMask) *IP {
+	return &IP{
+		IPNet: net.IPNet{
+			IP:   ip,
+			Mask: mask,
+		},
+	}
+}
+
+// ParseIP will parse string s as an IP, and return it.
+// The string s must be formed like <ip>[/<prefix>].
+// If s is not a valid textual representation of an IP,
+// will return nil.
+func ParseIP(s string) *IP {
+	if strings.ContainsAny(s, "/") {
+		ip, ipNet, err := net.ParseCIDR(s)
+		if err != nil {
+			return nil
+		}
+		return newIP(ip, ipNet.Mask)
+	} else {
+		ip := net.ParseIP(s)
+		if ip == nil {
+			return nil
+		}
+		return newIP(ip, nil)
+	}
+}
+
+// ToIP will return a net.IP in standard form from this IP.
+// If this IP can not be converted to a valid net.IP, will return nil.
+func (i *IP) ToIP() net.IP {
+	switch {
+	case i.IP.To4() != nil:
+		return i.IP.To4()
+	case i.IP.To16() != nil:
+		return i.IP.To16()
+	default:
+		return nil
+	}
+}
+
+// String returns the string form of this IP.
+func (i *IP) String() string {
+	if len(i.Mask) > 0 {
+		return i.IPNet.String()
+	}
+	return i.IP.String()
+}
+
+// MarshalText implements the encoding.TextMarshaler interface.
+// The encoding is the same as returned by String,
+// But when len(ip) is zero, will return an empty slice.
+func (i *IP) MarshalText() ([]byte, error) {
+	if len(i.IP) == 0 {
+		return []byte{}, nil
+	}
+	return []byte(i.String()), nil
+}
+
+// UnmarshalText implements the encoding.TextUnmarshaler interface.
+// The textual bytes are expected in a form accepted by Parse,
+// But when len(b) is zero, will return an empty IP.
+func (i *IP) UnmarshalText(b []byte) error {
+	if len(b) == 0 {
+		*i = IP{}
+		return nil
+	}
+
+	ip := ParseIP(string(b))
+	if ip == nil {
+		return fmt.Errorf("invalid IP address %s", string(b))
+	}
+	*i = *ip
+	return nil
+}

pkg/ip/ip_test.go

@@ -0,0 +1,272 @@
+// Copyright 2021 CNI authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package ip
+
+import (
+	"encoding/json"
+	"fmt"
+	"net"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("IP Operations", func() {
+	It("Parse", func() {
+		testCases := []struct {
+			ipStr    string
+			expected *IP
+		}{
+			{
+				"192.168.0.10",
+				newIP(net.IPv4(192, 168, 0, 10), nil),
+			},
+			{
+				"2001:db8::1",
+				newIP(net.ParseIP("2001:db8::1"), nil),
+			},
+			{
+				"192.168.0.10/24",
+				newIP(net.IPv4(192, 168, 0, 10), net.IPv4Mask(255, 255, 255, 0)),
+			},
+			{
+				"2001:db8::1/64",
+				newIP(net.ParseIP("2001:db8::1"), net.CIDRMask(64, 128)),
+			},
+			{
+				"invalid",
+				nil,
+			},
+		}
+
+		for _, test := range testCases {
+			ip := ParseIP(test.ipStr)
+
+			Expect(ip).To(Equal(test.expected))
+		}
+	})
+
+	It("String", func() {
+		testCases := []struct {
+			ip       *IP
+			expected string
+		}{
+			{
+				newIP(net.IPv4(192, 168, 0, 1), net.IPv4Mask(255, 255, 255, 0)),
+				"192.168.0.1/24",
+			},
+			{
+				newIP(net.IPv4(192, 168, 0, 2), nil),
+				"192.168.0.2",
+			},
+			{
+				newIP(net.ParseIP("2001:db8::1"), nil),
+				"2001:db8::1",
+			},
+			{
+				newIP(net.ParseIP("2001:db8::1"), net.CIDRMask(64, 128)),
+				"2001:db8::1/64",
+			},
+			{
+				newIP(nil, nil),
+				"<nil>",
+			},
+		}
+
+		for _, test := range testCases {
+			Expect(test.ip.String()).To(Equal(test.expected))
+		}
+	})
+
+	It("ToIP", func() {
+		testCases := []struct {
+			ip          *IP
+			expectedLen int
+			expectedIP  net.IP
+		}{
+			{
+				newIP(net.IPv4(192, 168, 0, 1), net.IPv4Mask(255, 255, 255, 0)),
+				net.IPv4len,
+				net.IP{192, 168, 0, 1},
+			},
+			{
+				newIP(net.IPv4(192, 168, 0, 2), nil),
+				net.IPv4len,
+				net.IP{192, 168, 0, 2},
+			},
+			{
+				newIP(net.ParseIP("2001:db8::1"), nil),
+				net.IPv6len,
+				net.IP{32, 1, 13, 184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1},
+			},
+			{
+				newIP(net.ParseIP("2001:db8::1"), net.CIDRMask(64, 128)),
+				net.IPv6len,
+				net.IP{32, 1, 13, 184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1},
+			},
+			{
+				newIP(nil, nil),
+				0,
+				nil,
+			},
+		}
+
+		for _, test := range testCases {
+			Expect(len(test.ip.ToIP())).To(Equal(test.expectedLen))
+			Expect(test.ip.ToIP()).To(Equal(test.expectedIP))
+		}
+	})
+
+	It("Encode", func() {
+		testCases := []struct {
+			object   interface{}
+			expected string
+		}{
+			{
+				newIP(net.IPv4(192, 168, 0, 1), net.IPv4Mask(255, 255, 255, 0)),
+				`"192.168.0.1/24"`,
+			},
+			{
+				newIP(net.IPv4(192, 168, 0, 2), nil),
+				`"192.168.0.2"`,
+			},
+			{
+				newIP(net.ParseIP("2001:db8::1"), nil),
+				`"2001:db8::1"`,
+			},
+			{
+				newIP(net.ParseIP("2001:db8::1"), net.CIDRMask(64, 128)),
+				`"2001:db8::1/64"`,
+			},
+			{
+				newIP(nil, nil),
+				`""`,
+			},
+			{
+				[]*IP{
+					newIP(net.IPv4(192, 168, 0, 1), net.IPv4Mask(255, 255, 255, 0)),
+					newIP(net.IPv4(192, 168, 0, 2), nil),
+					newIP(net.ParseIP("2001:db8::1"), nil),
+					newIP(net.ParseIP("2001:db8::1"), net.CIDRMask(64, 128)),
+					newIP(nil, nil),
+				},
+				`["192.168.0.1/24","192.168.0.2","2001:db8::1","2001:db8::1/64",""]`,
+			},
+		}
+
+		for _, test := range testCases {
+			bytes, err := json.Marshal(test.object)
+
+			Expect(err).NotTo(HaveOccurred())
+			Expect(string(bytes)).To(Equal(test.expected))
+		}
+	})
+
+	It("Decode", func() {
+		Context("valid IP", func() {
+			testCases := []struct {
+				text     string
+				expected *IP
+			}{
+				{
+					`"192.168.0.1"`,
+					newIP(net.IPv4(192, 168, 0, 1), nil),
+				},
+				{
+					`"192.168.0.1/24"`,
+					newIP(net.IPv4(192, 168, 0, 1), net.IPv4Mask(255, 255, 255, 0)),
+				},
+				{
+					`"2001:db8::1"`,
+					newIP(net.ParseIP("2001:db8::1"), nil),
+				},
+				{
+					`"2001:db8::1/64"`,
+					newIP(net.ParseIP("2001:db8::1"), net.CIDRMask(64, 128)),
+				},
+			}
+
+			for _, test := range testCases {
+				ip := &IP{}
+				err := json.Unmarshal([]byte(test.text), ip)
+
+				Expect(err).NotTo(HaveOccurred())
+				Expect(ip).To(Equal(test.expected))
+			}
+
+		})
+
+		Context("empty text", func() {
+			ip := &IP{}
+			err := json.Unmarshal([]byte(`""`), ip)
+
+			Expect(err).NotTo(HaveOccurred())
+			Expect(ip).To(Equal(newIP(nil, nil)))
+		})
+
+		Context("invalid IP", func() {
+			testCases := []struct {
+				text        string
+				expectedErr error
+			}{
+				{
+					`"192.168.0.1000"`,
+					fmt.Errorf("invalid IP address 192.168.0.1000"),
+				},
+				{
+					`"2001:db8::1/256"`,
+					fmt.Errorf("invalid IP address 2001:db8::1/256"),
+				},
+				{
+					`"test"`,
+					fmt.Errorf("invalid IP address test"),
+				},
+			}
+
+			for _, test := range testCases {
+				err := json.Unmarshal([]byte(test.text), &IP{})
+
+				Expect(err).To(HaveOccurred())
+				Expect(err).To(Equal(test.expectedErr))
+			}
+		})
+
+		Context("IP slice", func() {
+			testCases := []struct {
+				text     string
+				expected []*IP
+			}{
+				{
+					`["192.168.0.1/24","192.168.0.2","2001:db8::1","2001:db8::1/64",""]`,
+					[]*IP{
+						newIP(net.IPv4(192, 168, 0, 1), net.IPv4Mask(255, 255, 255, 0)),
+						newIP(net.IPv4(192, 168, 0, 2), nil),
+						newIP(net.ParseIP("2001:db8::1"), nil),
+						newIP(net.ParseIP("2001:db8::1"), net.CIDRMask(64, 128)),
+						newIP(nil, nil),
+					},
+				},
+			}
+
+			for _, test := range testCases {
+				ips := make([]*IP, 0)
+				err := json.Unmarshal([]byte(test.text), &ips)
+
+				Expect(err).NotTo(HaveOccurred())
+				Expect(ips).To(Equal(test.expected))
+			}
+		})
+	})
+})

pkg/ip/ipforward_linux.go

@@ -16,9 +16,9 @@ package ip
 
 import (
 	"bytes"
-	"io/ioutil"
+	"os"
 
-	"github.com/containernetworking/cni/pkg/types/current"
+	current "github.com/containernetworking/cni/pkg/types/100"
 )
 
 func EnableIP4Forward() error {
@@ -36,12 +36,13 @@ func EnableForward(ips []*current.IPConfig) error {
 	v6 := false
 
 	for _, ip := range ips {
-		if ip.Version == "4" && !v4 {
+		isV4 := ip.Address.IP.To4() != nil
+		if isV4 && !v4 {
 			if err := EnableIP4Forward(); err != nil {
 				return err
 			}
 			v4 = true
-		} else if ip.Version == "6" && !v6 {
+		} else if !isV4 && !v6 {
 			if err := EnableIP6Forward(); err != nil {
 				return err
 			}
@@ -52,10 +53,10 @@ func EnableForward(ips []*current.IPConfig) error {
 }
 
 func echo1(f string) error {
-	if content, err := ioutil.ReadFile(f); err == nil {
+	if content, err := os.ReadFile(f); err == nil {
 		if bytes.Equal(bytes.TrimSpace(content), []byte("1")) {
 			return nil
 		}
 	}
-	return ioutil.WriteFile(f, []byte("1"), 0644)
+	return os.WriteFile(f, []byte("1"), 0644)
 }

pkg/ip/ipforward_linux_test.go

@@ -1,7 +1,6 @@
 package ip
 
 import (
-	"io/ioutil"
 	"os"
 	"time"
 
@@ -11,7 +10,7 @@ import (
 
 var _ = Describe("IpforwardLinux", func() {
 	It("echo1 must not write the file if content is 1", func() {
-		file, err := ioutil.TempFile(os.TempDir(), "containernetworking")
+		file, err := os.CreateTemp("", "containernetworking")
 		Expect(err).NotTo(HaveOccurred())
 		defer os.Remove(file.Name())
 		err = echo1(file.Name())

pkg/ip/link_linux.go

@@ -25,7 +25,6 @@ import (
 	"github.com/vishvananda/netlink"
 
 	"github.com/containernetworking/plugins/pkg/ns"
-	"github.com/containernetworking/plugins/pkg/utils/hwaddr"
 	"github.com/containernetworking/plugins/pkg/utils/sysctl"
 )
 
@@ -33,19 +32,27 @@ var (
 	ErrLinkNotFound = errors.New("link not found")
 )
 
-func makeVethPair(name, peer string, mtu int) (netlink.Link, error) {
+// makeVethPair is called from within the container's network namespace
+func makeVethPair(name, peer string, mtu int, mac string, hostNS ns.NetNS) (netlink.Link, error) {
 	veth := &netlink.Veth{
 		LinkAttrs: netlink.LinkAttrs{
-			Name:  name,
-			Flags: net.FlagUp,
-			MTU:   mtu,
+			Name: name,
+			MTU:  mtu,
 		},
-		PeerName: peer,
+		PeerName:      peer,
+		PeerNamespace: netlink.NsFd(int(hostNS.Fd())),
+	}
+	if mac != "" {
+		m, err := net.ParseMAC(mac)
+		if err != nil {
+			return nil, err
+		}
+		veth.LinkAttrs.HardwareAddr = m
 	}
 	if err := netlink.LinkAdd(veth); err != nil {
 		return nil, err
 	}
-	// Re-fetch the link to get its creation-time parameters, e.g. index and mac
+	// Re-fetch the container link to get its creation-time parameters, e.g. index and mac
 	veth2, err := netlink.LinkByName(name)
 	if err != nil {
 		netlink.LinkDel(veth) // try and clean up the link if possible.
@@ -62,7 +69,7 @@ func peerExists(name string) bool {
 	return true
 }
 
-func makeVeth(name, vethPeerName string, mtu int) (peerName string, veth netlink.Link, err error) {
+func makeVeth(name, vethPeerName string, mtu int, mac string, hostNS ns.NetNS) (peerName string, veth netlink.Link, err error) {
 	for i := 0; i < 10; i++ {
 		if vethPeerName != "" {
 			peerName = vethPeerName
@@ -73,7 +80,7 @@ func makeVeth(name, vethPeerName string, mtu int) (peerName string, veth netlink
 			}
 		}
 
-		veth, err = makeVethPair(name, peerName, mtu)
+		veth, err = makeVethPair(name, peerName, mtu, mac, hostNS)
 		switch {
 		case err == nil:
 			return
@@ -99,7 +106,7 @@ func makeVeth(name, vethPeerName string, mtu int) (peerName string, veth netlink
 // RandomVethName returns string "veth" with random prefix (hashed from entropy)
 func RandomVethName() (string, error) {
 	entropy := make([]byte, 4)
-	_, err := rand.Reader.Read(entropy)
+	_, err := rand.Read(entropy)
 	if err != nil {
 		return "", fmt.Errorf("failed to generate random veth name: %v", err)
 	}
@@ -132,25 +139,13 @@ func ifaceFromNetlinkLink(l netlink.Link) net.Interface {
 // devices and move the host-side veth into the provided hostNS namespace.
 // hostVethName: If hostVethName is not specified, the host-side veth name will use a random string.
 // On success, SetupVethWithName returns (hostVeth, containerVeth, nil)
-func SetupVethWithName(contVethName, hostVethName string, mtu int, hostNS ns.NetNS) (net.Interface, net.Interface, error) {
-	hostVethName, contVeth, err := makeVeth(contVethName, hostVethName, mtu)
+func SetupVethWithName(contVethName, hostVethName string, mtu int, contVethMac string, hostNS ns.NetNS) (net.Interface, net.Interface, error) {
+	hostVethName, contVeth, err := makeVeth(contVethName, hostVethName, mtu, contVethMac, hostNS)
 	if err != nil {
 		return net.Interface{}, net.Interface{}, err
 	}
 
-	if err = netlink.LinkSetUp(contVeth); err != nil {
-		return net.Interface{}, net.Interface{}, fmt.Errorf("failed to set %q up: %v", contVethName, err)
-	}
-
-	hostVeth, err := netlink.LinkByName(hostVethName)
-	if err != nil {
-		return net.Interface{}, net.Interface{}, fmt.Errorf("failed to lookup %q: %v", hostVethName, err)
-	}
-
-	if err = netlink.LinkSetNsFd(hostVeth, int(hostNS.Fd())); err != nil {
-		return net.Interface{}, net.Interface{}, fmt.Errorf("failed to move veth to host netns: %v", err)
-	}
-
+	var hostVeth netlink.Link
 	err = hostNS.Do(func(_ ns.NetNS) error {
 		hostVeth, err = netlink.LinkByName(hostVethName)
 		if err != nil {
@@ -175,8 +170,8 @@ func SetupVethWithName(contVethName, hostVethName string, mtu int, hostNS ns.Net
 // Call SetupVeth from inside the container netns.  It will create both veth
 // devices and move the host-side veth into the provided hostNS namespace.
 // On success, SetupVeth returns (hostVeth, containerVeth, nil)
-func SetupVeth(contVethName string, mtu int, hostNS ns.NetNS) (net.Interface, net.Interface, error) {
-	return SetupVethWithName(contVethName, "", mtu, hostNS)
+func SetupVeth(contVethName string, mtu int, contVethMac string, hostNS ns.NetNS) (net.Interface, net.Interface, error) {
+	return SetupVethWithName(contVethName, "", mtu, contVethMac, hostNS)
 }
 
 // DelLinkByName removes an interface link.
@@ -225,33 +220,6 @@ func DelLinkByNameAddr(ifName string) ([]*net.IPNet, error) {
 	return out, nil
 }
 
-func SetHWAddrByIP(ifName string, ip4 net.IP, ip6 net.IP) error {
-	iface, err := netlink.LinkByName(ifName)
-	if err != nil {
-		return fmt.Errorf("failed to lookup %q: %v", ifName, err)
-	}
-
-	switch {
-	case ip4 == nil && ip6 == nil:
-		return fmt.Errorf("neither ip4 or ip6 specified")
-
-	case ip4 != nil:
-		{
-			hwAddr, err := hwaddr.GenerateHardwareAddr4(ip4, hwaddr.PrivateMACPrefix)
-			if err != nil {
-				return fmt.Errorf("failed to generate hardware addr: %v", err)
-			}
-			if err = netlink.LinkSetHardwareAddr(iface, hwAddr); err != nil {
-				return fmt.Errorf("failed to add hardware addr to %q: %v", ifName, err)
-			}
-		}
-	case ip6 != nil:
-		// TODO: IPv6
-	}
-
-	return nil
-}
-
 // GetVethPeerIfindex returns the veth link object, the peer ifindex of the
 // veth, or an error. This peer ifindex will only be valid in the peer's
 // network namespace.

pkg/ip/link_linux_test.go

@@ -51,8 +51,6 @@ var _ = Describe("Link", func() {
 		hostVethName      string
 		containerVethName string
 
-		ip4one             = net.ParseIP("1.1.1.1")
-		ip4two             = net.ParseIP("1.1.1.2")
 		originalRandReader = rand.Reader
 	)
 
@@ -72,7 +70,7 @@ var _ = Describe("Link", func() {
 		_ = containerNetNS.Do(func(ns.NetNS) error {
 			defer GinkgoRecover()
 
-			hostVeth, containerVeth, err = ip.SetupVeth(fmt.Sprintf(ifaceFormatString, ifaceCounter), mtu, hostNetNS)
+			hostVeth, containerVeth, err = ip.SetupVeth(fmt.Sprintf(ifaceFormatString, ifaceCounter), mtu, "", hostNetNS)
 			if err != nil {
 				return err
 			}
@@ -159,7 +157,7 @@ var _ = Describe("Link", func() {
 			_ = containerNetNS.Do(func(ns.NetNS) error {
 				defer GinkgoRecover()
 
-				_, _, err := ip.SetupVeth(containerVethName, mtu, hostNetNS)
+				_, _, err := ip.SetupVeth(containerVethName, mtu, "", hostNetNS)
 				Expect(err.Error()).To(Equal(fmt.Sprintf("container veth name provided (%s) already exists", containerVethName)))
 
 				return nil
@@ -189,9 +187,9 @@ var _ = Describe("Link", func() {
 		It("returns useful error", func() {
 			_ = containerNetNS.Do(func(ns.NetNS) error {
 				defer GinkgoRecover()
-				_, _, err := ip.SetupVeth(containerVethName, mtu, hostNetNS)
-				Expect(err.Error()).To(HavePrefix("failed to move veth to host netns: "))
-
+				_, _, err := ip.SetupVeth(containerVethName, mtu, "", hostNetNS)
+				Expect(err.Error()).To(HavePrefix("container veth name provided"))
+				Expect(err.Error()).To(HaveSuffix("already exists"))
 				return nil
 			})
 		})
@@ -207,7 +205,7 @@ var _ = Describe("Link", func() {
 			_ = containerNetNS.Do(func(ns.NetNS) error {
 				defer GinkgoRecover()
 
-				hostVeth, _, err := ip.SetupVeth(containerVethName, mtu, hostNetNS)
+				hostVeth, _, err := ip.SetupVeth(containerVethName, mtu, "", hostNetNS)
 				Expect(err).NotTo(HaveOccurred())
 				hostVethName = hostVeth.Name
 				return nil
@@ -233,6 +231,32 @@ var _ = Describe("Link", func() {
 			})
 		})
 
+		It("successfully creates a veth pair with an explicit mac", func() {
+			const mac = "02:00:00:00:01:23"
+			_ = containerNetNS.Do(func(ns.NetNS) error {
+				defer GinkgoRecover()
+
+				hostVeth, _, err := ip.SetupVeth(containerVethName, mtu, mac, hostNetNS)
+				Expect(err).NotTo(HaveOccurred())
+				hostVethName = hostVeth.Name
+
+				link, err := netlink.LinkByName(containerVethName)
+				Expect(err).NotTo(HaveOccurred())
+				Expect(link.Attrs().HardwareAddr.String()).To(Equal(mac))
+
+				return nil
+			})
+
+			_ = hostNetNS.Do(func(ns.NetNS) error {
+				defer GinkgoRecover()
+
+				link, err := netlink.LinkByName(hostVethName)
+				Expect(err).NotTo(HaveOccurred())
+				Expect(link.Attrs().HardwareAddr.String()).NotTo(Equal(mac))
+
+				return nil
+			})
+		})
 	})
 
 	It("DelLinkByName must delete the veth endpoints", func() {
@@ -270,41 +294,4 @@ var _ = Describe("Link", func() {
 			return nil
 		})
 	})
-
-	It("SetHWAddrByIP must change the interface hwaddr and be predictable", func() {
-
-		_ = containerNetNS.Do(func(ns.NetNS) error {
-			defer GinkgoRecover()
-
-			var err error
-			hwaddrBefore := getHwAddr(containerVethName)
-
-			err = ip.SetHWAddrByIP(containerVethName, ip4one, nil)
-			Expect(err).NotTo(HaveOccurred())
-			hwaddrAfter1 := getHwAddr(containerVethName)
-
-			Expect(hwaddrBefore).NotTo(Equal(hwaddrAfter1))
-			Expect(hwaddrAfter1).To(Equal(ip4onehwaddr))
-
-			return nil
-		})
-	})
-
-	It("SetHWAddrByIP must be injective", func() {
-
-		_ = containerNetNS.Do(func(ns.NetNS) error {
-			defer GinkgoRecover()
-
-			err := ip.SetHWAddrByIP(containerVethName, ip4one, nil)
-			Expect(err).NotTo(HaveOccurred())
-			hwaddrAfter1 := getHwAddr(containerVethName)
-
-			err = ip.SetHWAddrByIP(containerVethName, ip4two, nil)
-			Expect(err).NotTo(HaveOccurred())
-			hwaddrAfter2 := getHwAddr(containerVethName)
-
-			Expect(hwaddrAfter1).NotTo(Equal(hwaddrAfter2))
-			return nil
-		})
-	})
 })

pkg/ip/route_linux.go

@@ -42,6 +42,11 @@ func AddHostRoute(ipn *net.IPNet, gw net.IP, dev netlink.Link) error {
 
 // AddDefaultRoute sets the default route on the given gateway.
 func AddDefaultRoute(gw net.IP, dev netlink.Link) error {
-	_, defNet, _ := net.ParseCIDR("0.0.0.0/0")
+	var defNet *net.IPNet
+	if gw.To4() != nil {
+		_, defNet, _ = net.ParseCIDR("0.0.0.0/0")
+	} else {
+		_, defNet, _ = net.ParseCIDR("::/0")
+	}
 	return AddRoute(defNet, gw, dev)
 }

pkg/ip/utils_linux.go

@@ -1,3 +1,4 @@
+//go:build linux
 // +build linux
 
 // Copyright 2016 CNI authors
@@ -21,7 +22,7 @@ import (
 	"net"
 
 	"github.com/containernetworking/cni/pkg/types"
-	"github.com/containernetworking/cni/pkg/types/current"
+	current "github.com/containernetworking/cni/pkg/types/100"
 	"github.com/vishvananda/netlink"
 )
 
@@ -57,15 +58,10 @@ func ValidateExpectedInterfaceIPs(ifName string, resultIPs []*current.IPConfig)
 
 		findGwy := &netlink.Route{Dst: ourPrefix}
 		routeFilter := netlink.RT_FILTER_DST
-		var family int
 
-		switch {
-		case ips.Version == "4":
+		family := netlink.FAMILY_V6
+		if ips.Address.IP.To4() != nil {
 			family = netlink.FAMILY_V4
-		case ips.Version == "6":
-			family = netlink.FAMILY_V6
-		default:
-			return fmt.Errorf("Invalid IP Version %v for interface %v", ips.Version, ifName)
 		}
 
 		gwy, err := netlink.RouteListFiltered(family, findGwy, routeFilter)

pkg/ipam/ipam_linux.go

@@ -19,7 +19,7 @@ import (
 	"net"
 	"os"
 
-	"github.com/containernetworking/cni/pkg/types/current"
+	current "github.com/containernetworking/cni/pkg/types/100"
 	"github.com/containernetworking/plugins/pkg/ip"
 	"github.com/containernetworking/plugins/pkg/utils/sysctl"
 
@@ -27,7 +27,8 @@ import (
 )
 
 const (
-	DisableIPv6SysctlTemplate = "net.ipv6.conf.%s.disable_ipv6"
+	// Note: use slash as separator so we can have dots in interface name (VLANs)
+	DisableIPv6SysctlTemplate = "net/ipv6/conf/%s/disable_ipv6"
 )
 
 // ConfigureIface takes the result of IPAM plugin and
@@ -42,10 +43,6 @@ func ConfigureIface(ifName string, res *current.Result) error {
 		return fmt.Errorf("failed to lookup %q: %v", ifName, err)
 	}
 
-	if err := netlink.LinkSetUp(link); err != nil {
-		return fmt.Errorf("failed to set %q UP: %v", ifName, err)
-	}
-
 	var v4gw, v6gw net.IP
 	var has_enabled_ipv6 bool = false
 	for _, ipc := range res.IPs {
@@ -60,7 +57,7 @@ func ConfigureIface(ifName string, res *current.Result) error {
 
 		// Make sure sysctl "disable_ipv6" is 0 if we are about to add
 		// an IPv6 address to the interface
-		if !has_enabled_ipv6 && ipc.Version == "6" {
+		if !has_enabled_ipv6 && ipc.Address.IP.To4() == nil {
 			// Enabled IPv6 for loopback "lo" and the interface
 			// being configured
 			for _, iface := range [2]string{"lo", ifName} {
@@ -68,8 +65,11 @@ func ConfigureIface(ifName string, res *current.Result) error {
 
 				// Read current sysctl value
 				value, err := sysctl.Sysctl(ipv6SysctlValueName)
-				if err != nil || value == "0" {
-					// FIXME: log warning if unable to read sysctl value
+				if err != nil {
+					fmt.Fprintf(os.Stderr, "ipam_linux: failed to read sysctl %q: %v\n", ipv6SysctlValueName, err)
+					continue
+				}
+				if value == "0" {
 					continue
 				}
 
@@ -95,6 +95,10 @@ func ConfigureIface(ifName string, res *current.Result) error {
 		}
 	}
 
+	if err := netlink.LinkSetUp(link); err != nil {
+		return fmt.Errorf("failed to set %q UP: %v", ifName, err)
+	}
+
 	if v6gw != nil {
 		ip.SettleAddresses(ifName, 10)
 	}
@@ -109,11 +113,14 @@ func ConfigureIface(ifName string, res *current.Result) error {
 				gw = v6gw
 			}
 		}
-		if err = ip.AddRoute(&r.Dst, gw, link); err != nil {
-			// we skip over duplicate routes as we assume the first one wins
-			if !os.IsExist(err) {
-				return fmt.Errorf("failed to add route '%v via %v dev %v': %v", r.Dst, gw, ifName, err)
-			}
+		route := netlink.Route{
+			Dst:       &r.Dst,
+			LinkIndex: link.Attrs().Index,
+			Gw:        gw,
+		}
+
+		if err = netlink.RouteAddEcmp(&route); err != nil {
+			return fmt.Errorf("failed to add route '%v via %v dev %v': %v", r.Dst, gw, ifName, err)
 		}
 	}
 

pkg/ipam/ipam_linux_test.go

@@ -19,7 +19,7 @@ import (
 	"syscall"
 
 	"github.com/containernetworking/cni/pkg/types"
-	"github.com/containernetworking/cni/pkg/types/current"
+	current "github.com/containernetworking/cni/pkg/types/100"
 	"github.com/containernetworking/plugins/pkg/ns"
 	"github.com/containernetworking/plugins/pkg/testutils"
 
@@ -109,13 +109,11 @@ var _ = Describe("ConfigureIface", func() {
 			},
 			IPs: []*current.IPConfig{
 				{
-					Version:   "4",
 					Interface: current.Int(0),
 					Address:   *ipv4,
 					Gateway:   ipgw4,
 				},
 				{
-					Version:   "6",
 					Interface: current.Int(0),
 					Address:   *ipv6,
 					Gateway:   ipgw6,
@@ -281,12 +279,10 @@ var _ = Describe("ConfigureIface", func() {
 			},
 			IPs: []*current.IPConfig{
 				{
-					Version: "4",
 					Address: *ipv4,
 					Gateway: ipgw4,
 				},
 				{
-					Version: "6",
 					Address: *ipv6,
 					Gateway: ipgw6,
 				},

pkg/link/link_suite_test.go

@@ -1,4 +1,4 @@
-// Copyright 2015 CNI authors
+// Copyright 2021 CNI authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -11,16 +11,17 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-package main
+
+package link_test
 
 import (
+	"testing"
+
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
-
-	"testing"
 )
 
-func TestFlannel(t *testing.T) {
+func TestIp(t *testing.T) {
 	RegisterFailHandler(Fail)
-	RunSpecs(t, "plugins/meta/flannel")
+	RunSpecs(t, "pkg/link")
 }

pkg/link/spoofcheck.go

@@ -0,0 +1,245 @@
+// Copyright 2021 CNI authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package link
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/networkplumbing/go-nft/nft"
+	"github.com/networkplumbing/go-nft/nft/schema"
+)
+
+const (
+	natTableName            = "nat"
+	preRoutingBaseChainName = "PREROUTING"
+)
+
+type NftConfigurer interface {
+	Apply(*nft.Config) error
+	Read() (*nft.Config, error)
+}
+
+type SpoofChecker struct {
+	iface      string
+	macAddress string
+	refID      string
+	configurer NftConfigurer
+}
+
+type defaultNftConfigurer struct{}
+
+func (_ defaultNftConfigurer) Apply(cfg *nft.Config) error {
+	return nft.ApplyConfig(cfg)
+}
+
+func (_ defaultNftConfigurer) Read() (*nft.Config, error) {
+	return nft.ReadConfig()
+}
+
+func NewSpoofChecker(iface, macAddress, refID string) *SpoofChecker {
+	return NewSpoofCheckerWithConfigurer(iface, macAddress, refID, defaultNftConfigurer{})
+}
+
+func NewSpoofCheckerWithConfigurer(iface, macAddress, refID string, configurer NftConfigurer) *SpoofChecker {
+	return &SpoofChecker{iface, macAddress, refID, configurer}
+}
+
+// Setup applies nftables configuration to restrict traffic
+// from the provided interface. Only traffic with the mentioned mac address
+// is allowed to pass, all others are blocked.
+// The configuration follows the format libvirt and ebtables implemented, allowing
+// extensions to the rules in the future.
+// refID is used to label the rules with a unique comment, identifying the rule-set.
+//
+// In order to take advantage of the nftables configuration change atomicity, the
+// following steps are taken to apply the configuration:
+// - Declare the table and chains (they will be created in case not present).
+// - Apply the rules, while first flushing the iface/mac specific regular chain rules.
+// Two transactions are used because the flush succeeds only if the table/chain it targets
+// exists. This avoids the need to query the existing state and acting upon it (a raceful pattern).
+// Although two transactions are taken place, only the 2nd one where the rules
+// are added has a real impact on the system.
+func (sc *SpoofChecker) Setup() error {
+	baseConfig := nft.NewConfig()
+
+	baseConfig.AddTable(&schema.Table{Family: schema.FamilyBridge, Name: natTableName})
+
+	baseConfig.AddChain(sc.baseChain())
+	ifaceChain := sc.ifaceChain()
+	baseConfig.AddChain(ifaceChain)
+	macChain := sc.macChain(ifaceChain.Name)
+	baseConfig.AddChain(macChain)
+
+	if err := sc.configurer.Apply(baseConfig); err != nil {
+		return fmt.Errorf("failed to setup spoof-check: %v", err)
+	}
+
+	rulesConfig := nft.NewConfig()
+
+	rulesConfig.FlushChain(ifaceChain)
+	rulesConfig.FlushChain(macChain)
+
+	rulesConfig.AddRule(sc.matchIfaceJumpToChainRule(preRoutingBaseChainName, ifaceChain.Name))
+	rulesConfig.AddRule(sc.jumpToChainRule(ifaceChain.Name, macChain.Name))
+	rulesConfig.AddRule(sc.matchMacRule(macChain.Name))
+	rulesConfig.AddRule(sc.dropRule(macChain.Name))
+
+	if err := sc.configurer.Apply(rulesConfig); err != nil {
+		return fmt.Errorf("failed to setup spoof-check: %v", err)
+	}
+
+	return nil
+}
+
+// Teardown removes the interface and mac-address specific chains and their rules.
+// The table and base-chain are expected to survive while the base-chain rule that matches the
+// interface is removed.
+func (sc *SpoofChecker) Teardown() error {
+	ifaceChain := sc.ifaceChain()
+	currentConfig, ifaceMatchRuleErr := sc.configurer.Read()
+	if ifaceMatchRuleErr == nil {
+		expectedRuleToFind := sc.matchIfaceJumpToChainRule(preRoutingBaseChainName, ifaceChain.Name)
+		// It is safer to exclude the statement matching, avoiding cases where a current statement includes
+		// additional default entries (e.g. counters).
+		ruleToFindExcludingStatements := *expectedRuleToFind
+		ruleToFindExcludingStatements.Expr = nil
+		rules := currentConfig.LookupRule(&ruleToFindExcludingStatements)
+		if len(rules) > 0 {
+			c := nft.NewConfig()
+			for _, rule := range rules {
+				c.DeleteRule(rule)
+			}
+			if err := sc.configurer.Apply(c); err != nil {
+				ifaceMatchRuleErr = fmt.Errorf("failed to delete iface match rule: %v", err)
+			}
+		} else {
+			fmt.Fprintf(os.Stderr, "spoofcheck/teardown: unable to detect iface match rule for deletion: %+v", expectedRuleToFind)
+		}
+	}
+
+	regularChainsConfig := nft.NewConfig()
+	regularChainsConfig.DeleteChain(ifaceChain)
+	regularChainsConfig.DeleteChain(sc.macChain(ifaceChain.Name))
+
+	var regularChainsErr error
+	if err := sc.configurer.Apply(regularChainsConfig); err != nil {
+		regularChainsErr = fmt.Errorf("failed to delete regular chains: %v", err)
+	}
+
+	if ifaceMatchRuleErr != nil || regularChainsErr != nil {
+		return fmt.Errorf("failed to teardown spoof-check: %v, %v", ifaceMatchRuleErr, regularChainsErr)
+	}
+	return nil
+}
+
+func (sc *SpoofChecker) matchIfaceJumpToChainRule(chain, toChain string) *schema.Rule {
+	return &schema.Rule{
+		Family: schema.FamilyBridge,
+		Table:  natTableName,
+		Chain:  chain,
+		Expr: []schema.Statement{
+			{Match: &schema.Match{
+				Op:    schema.OperEQ,
+				Left:  schema.Expression{RowData: []byte(`{"meta":{"key":"iifname"}}`)},
+				Right: schema.Expression{String: &sc.iface},
+			}},
+			{Verdict: schema.Verdict{Jump: &schema.ToTarget{Target: toChain}}},
+		},
+		Comment: ruleComment(sc.refID),
+	}
+}
+
+func (sc *SpoofChecker) jumpToChainRule(chain, toChain string) *schema.Rule {
+	return &schema.Rule{
+		Family: schema.FamilyBridge,
+		Table:  natTableName,
+		Chain:  chain,
+		Expr: []schema.Statement{
+			{Verdict: schema.Verdict{Jump: &schema.ToTarget{Target: toChain}}},
+		},
+		Comment: ruleComment(sc.refID),
+	}
+}
+
+func (sc *SpoofChecker) matchMacRule(chain string) *schema.Rule {
+	return &schema.Rule{
+		Family: schema.FamilyBridge,
+		Table:  natTableName,
+		Chain:  chain,
+		Expr: []schema.Statement{
+			{Match: &schema.Match{
+				Op: schema.OperEQ,
+				Left: schema.Expression{Payload: &schema.Payload{
+					Protocol: schema.PayloadProtocolEther,
+					Field:    schema.PayloadFieldEtherSAddr,
+				}},
+				Right: schema.Expression{String: &sc.macAddress},
+			}},
+			{Verdict: schema.Verdict{SimpleVerdict: schema.SimpleVerdict{Return: true}}},
+		},
+		Comment: ruleComment(sc.refID),
+	}
+}
+
+func (sc *SpoofChecker) dropRule(chain string) *schema.Rule {
+	macRulesIndex := nft.NewRuleIndex()
+	return &schema.Rule{
+		Family: schema.FamilyBridge,
+		Table:  natTableName,
+		Chain:  chain,
+		Index:  macRulesIndex.Next(),
+		Expr: []schema.Statement{
+			{Verdict: schema.Verdict{SimpleVerdict: schema.SimpleVerdict{Drop: true}}},
+		},
+		Comment: ruleComment(sc.refID),
+	}
+}
+
+func (_ *SpoofChecker) baseChain() *schema.Chain {
+	chainPriority := -300
+	return &schema.Chain{
+		Family: schema.FamilyBridge,
+		Table:  natTableName,
+		Name:   preRoutingBaseChainName,
+		Type:   schema.TypeFilter,
+		Hook:   schema.HookPreRouting,
+		Prio:   &chainPriority,
+		Policy: schema.PolicyAccept,
+	}
+}
+
+func (sc *SpoofChecker) ifaceChain() *schema.Chain {
+	ifaceChainName := "cni-br-iface-" + sc.refID
+	return &schema.Chain{
+		Family: schema.FamilyBridge,
+		Table:  natTableName,
+		Name:   ifaceChainName,
+	}
+}
+
+func (_ *SpoofChecker) macChain(ifaceChainName string) *schema.Chain {
+	macChainName := ifaceChainName + "-mac"
+	return &schema.Chain{
+		Family: schema.FamilyBridge,
+		Table:  natTableName,
+		Name:   macChainName,
+	}
+}
+
+func ruleComment(id string) string {
+	const refIDPrefix = "macspoofchk-"
+	return refIDPrefix + id
+}

pkg/link/spoofcheck_test.go

@@ -0,0 +1,297 @@
+// Copyright 2021 CNI authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package link_test
+
+import (
+	"fmt"
+	"github.com/networkplumbing/go-nft/nft"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/containernetworking/plugins/pkg/link"
+)
+
+var _ = Describe("spoofcheck", func() {
+	iface := "net0"
+	mac := "02:00:00:00:12:34"
+	id := "container99-net1"
+
+	Context("setup", func() {
+		It("succeeds", func() {
+			c := configurerStub{}
+			sc := link.NewSpoofCheckerWithConfigurer(iface, mac, id, &c)
+			Expect(sc.Setup()).To(Succeed())
+
+			assertExpectedTableAndChainsInSetupConfig(c)
+			assertExpectedRulesInSetupConfig(c)
+		})
+
+		It("fails to setup config when 1st apply is unsuccessful (declare table and chains)", func() {
+			c := &configurerStub{failFirstApplyConfig: true}
+			sc := link.NewSpoofCheckerWithConfigurer(iface, mac, id, c)
+			Expect(sc.Setup()).To(MatchError("failed to setup spoof-check: " + errorFirstApplyText))
+		})
+
+		It("fails to setup config when 2nd apply is unsuccessful (flush and add the rules)", func() {
+			c := &configurerStub{failSecondApplyConfig: true}
+			sc := link.NewSpoofCheckerWithConfigurer(iface, mac, id, c)
+			Expect(sc.Setup()).To(MatchError("failed to setup spoof-check: " + errorSecondApplyText))
+		})
+	})
+
+	Context("teardown", func() {
+		It("succeeds", func() {
+			existingConfig := nft.NewConfig()
+			existingConfig.FromJSON([]byte(rowConfigWithRulesOnly()))
+			c := configurerStub{readConfig: existingConfig}
+
+			sc := link.NewSpoofCheckerWithConfigurer("", "", id, &c)
+			Expect(sc.Teardown()).To(Succeed())
+
+			assertExpectedBaseChainRuleDeletionInTeardownConfig(c)
+			assertExpectedRegularChainsDeletionInTeardownConfig(c)
+		})
+
+		It("fails, 1st apply is unsuccessful (delete iface match rule)", func() {
+			config := nft.NewConfig()
+			config.FromJSON([]byte(rowConfigWithRulesOnly()))
+			c := &configurerStub{applyConfig: []*nft.Config{config}, readConfig: config, failFirstApplyConfig: true}
+			sc := link.NewSpoofCheckerWithConfigurer("", "", id, c)
+			Expect(sc.Teardown()).To(MatchError(fmt.Sprintf(
+				"failed to teardown spoof-check: failed to delete iface match rule: %s, <nil>", errorFirstApplyText,
+			)))
+		})
+
+		It("fails, read current config is unsuccessful", func() {
+			config := nft.NewConfig()
+			config.FromJSON([]byte(rowConfigWithRulesOnly()))
+			c := &configurerStub{applyConfig: []*nft.Config{config}, readConfig: config, failReadConfig: true}
+			sc := link.NewSpoofCheckerWithConfigurer("", "", id, c)
+			Expect(sc.Teardown()).To(MatchError(fmt.Sprintf(
+				"failed to teardown spoof-check: %s, <nil>", errorReadText,
+			)))
+		})
+
+		It("fails, 2nd apply is unsuccessful (delete the regular chains)", func() {
+			config := nft.NewConfig()
+			config.FromJSON([]byte(rowConfigWithRulesOnly()))
+			c := &configurerStub{applyConfig: []*nft.Config{config}, readConfig: config, failSecondApplyConfig: true}
+			sc := link.NewSpoofCheckerWithConfigurer("", "", id, c)
+			Expect(sc.Teardown()).To(MatchError(fmt.Sprintf(
+				"failed to teardown spoof-check: <nil>, failed to delete regular chains: %s", errorSecondApplyText,
+			)))
+		})
+
+		It("fails, both applies are unsuccessful", func() {
+			config := nft.NewConfig()
+			config.FromJSON([]byte(rowConfigWithRulesOnly()))
+			c := &configurerStub{
+				applyConfig:           []*nft.Config{config},
+				readConfig:            config,
+				failFirstApplyConfig:  true,
+				failSecondApplyConfig: true,
+			}
+			sc := link.NewSpoofCheckerWithConfigurer("", "", id, c)
+			Expect(sc.Teardown()).To(MatchError(fmt.Sprintf(
+				"failed to teardown spoof-check: "+
+					"failed to delete iface match rule: %s, "+
+					"failed to delete regular chains: %s",
+				errorFirstApplyText, errorSecondApplyText,
+			)))
+		})
+	})
+})
+
+func assertExpectedRegularChainsDeletionInTeardownConfig(action configurerStub) {
+	deleteRegularChainRulesJsonConfig, err := action.applyConfig[1].ToJSON()
+	ExpectWithOffset(1, err).NotTo(HaveOccurred())
+
+	expectedDeleteRegularChainRulesJsonConfig := `
+			{"nftables": [
+				{"delete": {"chain": {
+					"family": "bridge",
+					"table": "nat",
+					"name": "cni-br-iface-container99-net1"
+				}}},
+				{"delete": {"chain": {
+					"family": "bridge",
+					"table": "nat",
+					"name": "cni-br-iface-container99-net1-mac"
+				}}}
+			]}`
+
+	ExpectWithOffset(1, string(deleteRegularChainRulesJsonConfig)).To(MatchJSON(expectedDeleteRegularChainRulesJsonConfig))
+}
+
+func assertExpectedBaseChainRuleDeletionInTeardownConfig(action configurerStub) {
+	deleteBaseChainRuleJsonConfig, err := action.applyConfig[0].ToJSON()
+	Expect(err).NotTo(HaveOccurred())
+
+	expectedDeleteIfaceMatchRuleJsonConfig := `
+            {"nftables": [
+				{"delete": {"rule": {
+					"family": "bridge",
+					"table": "nat",
+					"chain": "PREROUTING",
+					"expr": [
+						{"match": {
+							"op": "==",
+							"left": {"meta": {"key": "iifname"}},
+							"right": "net0"
+						}},
+						{"jump": {"target": "cni-br-iface-container99-net1"}}
+					],
+					"comment": "macspoofchk-container99-net1"
+				}}}
+			]}`
+	Expect(string(deleteBaseChainRuleJsonConfig)).To(MatchJSON(expectedDeleteIfaceMatchRuleJsonConfig))
+}
+
+func rowConfigWithRulesOnly() string {
+	return `
+            {"nftables":[
+                {"rule":{"family":"bridge","table":"nat","chain":"PREROUTING",
+                    "expr":[
+                        {"match":{"op":"==","left":{"meta":{"key":"iifname"}},"right":"net0"}},
+                        {"jump":{"target":"cni-br-iface-container99-net1"}}
+                    ],
+                    "comment":"macspoofchk-container99-net1"}},
+                {"rule":{"family":"bridge","table":"nat","chain":"cni-br-iface-container99-net1",
+                    "expr":[
+                        {"jump":{"target":"cni-br-iface-container99-net1-mac"}}
+                    ],
+                    "comment":"macspoofchk-container99-net1"}},
+                {"rule":{"family":"bridge","table":"nat","chain":"cni-br-iface-container99-net1-mac",
+                    "expr":[
+                        {"match":{
+                            "op":"==",
+                            "left":{"payload":{"protocol":"ether","field":"saddr"}},
+                            "right":"02:00:00:00:12:34"
+                        }},
+                        {"return":null}
+                    ],
+                    "comment":"macspoofchk-container99-net1"}},
+                {"rule":{"family":"bridge","table":"nat","chain":"cni-br-iface-container99-net1-mac",
+                    "expr":[{"drop":null}],
+                    "index":0,
+                    "comment":"macspoofchk-container99-net1"}}
+            ]}`
+}
+
+func assertExpectedTableAndChainsInSetupConfig(c configurerStub) {
+	config := c.applyConfig[0]
+	jsonConfig, err := config.ToJSON()
+	ExpectWithOffset(1, err).NotTo(HaveOccurred())
+
+	expectedConfig := `
+        {"nftables": [
+            {"table": {"family": "bridge", "name": "nat"}},
+            {"chain": {
+                "family": "bridge",
+                "table": "nat",
+                "name": "PREROUTING",
+                "type": "filter",
+                "hook": "prerouting",
+                "prio": -300,
+                "policy": "accept"
+            }},
+            {"chain": {
+                "family": "bridge",
+                "table": "nat",
+                "name": "cni-br-iface-container99-net1"
+            }},
+            {"chain": {
+                "family": "bridge",
+                "table": "nat",
+                "name": "cni-br-iface-container99-net1-mac"
+            }}
+        ]}`
+	ExpectWithOffset(1, string(jsonConfig)).To(MatchJSON(expectedConfig))
+}
+
+func assertExpectedRulesInSetupConfig(c configurerStub) {
+	config := c.applyConfig[1]
+	jsonConfig, err := config.ToJSON()
+	ExpectWithOffset(1, err).NotTo(HaveOccurred())
+
+	expectedConfig := `
+            {"nftables":[
+                {"flush":{"chain":{"family":"bridge","table":"nat","name":"cni-br-iface-container99-net1"}}},
+                {"flush":{"chain":{"family":"bridge","table":"nat","name":"cni-br-iface-container99-net1-mac"}}},
+                {"rule":{"family":"bridge","table":"nat","chain":"PREROUTING",
+                    "expr":[
+                        {"match":{"op":"==","left":{"meta":{"key":"iifname"}},"right":"net0"}},
+                        {"jump":{"target":"cni-br-iface-container99-net1"}}
+                    ],
+                    "comment":"macspoofchk-container99-net1"}},
+                {"rule":{"family":"bridge","table":"nat","chain":"cni-br-iface-container99-net1",
+                    "expr":[
+                        {"jump":{"target":"cni-br-iface-container99-net1-mac"}}
+                    ],
+                    "comment":"macspoofchk-container99-net1"}},
+                {"rule":{"family":"bridge","table":"nat","chain":"cni-br-iface-container99-net1-mac",
+                    "expr":[
+                        {"match":{
+                            "op":"==",
+                            "left":{"payload":{"protocol":"ether","field":"saddr"}},
+                            "right":"02:00:00:00:12:34"
+                        }},
+                        {"return":null}
+                    ],
+                    "comment":"macspoofchk-container99-net1"}},
+                {"rule":{"family":"bridge","table":"nat","chain":"cni-br-iface-container99-net1-mac",
+                    "expr":[{"drop":null}],
+                    "index":0,
+                    "comment":"macspoofchk-container99-net1"}}
+            ]}`
+	ExpectWithOffset(1, string(jsonConfig)).To(MatchJSON(expectedConfig))
+}
+
+const (
+	errorFirstApplyText  = "1st apply failed"
+	errorSecondApplyText = "2nd apply failed"
+	errorReadText        = "read failed"
+)
+
+type configurerStub struct {
+	applyConfig []*nft.Config
+	readConfig  *nft.Config
+
+	applyCounter int
+
+	failFirstApplyConfig  bool
+	failSecondApplyConfig bool
+	failReadConfig        bool
+}
+
+func (a *configurerStub) Apply(c *nft.Config) error {
+	a.applyCounter++
+	if a.failFirstApplyConfig && a.applyCounter == 1 {
+		return fmt.Errorf(errorFirstApplyText)
+	}
+	if a.failSecondApplyConfig && a.applyCounter == 2 {
+		return fmt.Errorf(errorSecondApplyText)
+	}
+	a.applyConfig = append(a.applyConfig, c)
+	return nil
+}
+
+func (a *configurerStub) Read() (*nft.Config, error) {
+	if a.failReadConfig {
+		return nil, fmt.Errorf(errorReadText)
+	}
+	return a.readConfig, nil
+}

pkg/ns/ns_linux.go

@@ -106,8 +106,8 @@ var _ NetNS = &netNS{}
 
 const (
 	// https://github.com/torvalds/linux/blob/master/include/uapi/linux/magic.h
-	NSFS_MAGIC   = 0x6e736673
-	PROCFS_MAGIC = 0x9fa0
+	NSFS_MAGIC   = unix.NSFS_MAGIC
+	PROCFS_MAGIC = unix.PROC_SUPER_MAGIC
 )
 
 type NSPathNotExistErr struct{ msg string }

pkg/ns/ns_linux_test.go

@@ -17,7 +17,6 @@ package ns_test
 import (
 	"errors"
 	"fmt"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 	"sync"
@@ -208,7 +207,7 @@ var _ = Describe("Linux namespace operations", func() {
 			})
 
 			It("fails when the path is not a namespace", func() {
-				tempFile, err := ioutil.TempFile("", "nstest")
+				tempFile, err := os.CreateTemp("", "nstest")
 				Expect(err).NotTo(HaveOccurred())
 				defer tempFile.Close()
 
@@ -262,7 +261,7 @@ var _ = Describe("Linux namespace operations", func() {
 		})
 
 		It("should refuse other paths", func() {
-			tempFile, err := ioutil.TempFile("", "nstest")
+			tempFile, err := os.CreateTemp("", "nstest")
 			Expect(err).NotTo(HaveOccurred())
 			defer tempFile.Close()
 

pkg/testutils/cmd.go

@@ -15,7 +15,7 @@
 package testutils
 
 import (
-	"io/ioutil"
+	"io"
 	"os"
 
 	"github.com/containernetworking/cni/pkg/skel"
@@ -52,7 +52,7 @@ func CmdAdd(cniNetns, cniContainerID, cniIfname string, conf []byte, f func() er
 
 	var out []byte
 	if err == nil {
-		out, err = ioutil.ReadAll(r)
+		out, err = io.ReadAll(r)
 	}
 	os.Stdout = oldStdout
 

pkg/testutils/dns.go

@@ -16,7 +16,6 @@ package testutils
 
 import (
 	"fmt"
-	"io/ioutil"
 	"os"
 	"strings"
 
@@ -28,7 +27,7 @@ import (
 // an error if any occurs while creating/writing the file. It is the caller's
 // responsibility to remove the file.
 func TmpResolvConf(dnsConf types.DNS) (string, error) {
-	f, err := ioutil.TempFile("", "cni_test_resolv.conf")
+	f, err := os.CreateTemp("", "cni_test_resolv.conf")
 	if err != nil {
 		return "", fmt.Errorf("failed to get temp file for CNI test resolv.conf: %v", err)
 	}

pkg/testutils/echo/echo_test.go

@@ -2,7 +2,7 @@ package main_test
 
 import (
 	"fmt"
-	"io/ioutil"
+	"io"
 	"net"
 	"os/exec"
 	"strings"
@@ -74,7 +74,7 @@ var _ = Describe("Echosvr", func() {
 			defer conn.Close()
 
 			fmt.Fprintf(conn, "hello\n")
-			Expect(ioutil.ReadAll(conn)).To(Equal([]byte("hello")))
+			Expect(io.ReadAll(conn)).To(Equal([]byte("hello")))
 		})
 	})
 

pkg/testutils/netns_linux.go

@@ -53,7 +53,7 @@ func NewNS() (ns.NetNS, error) {
 	nsRunDir := getNsRunDir()
 
 	b := make([]byte, 16)
-	_, err := rand.Reader.Read(b)
+	_, err := rand.Read(b)
 	if err != nil {
 		return nil, fmt.Errorf("failed to generate random netns name: %v", err)
 	}

pkg/testutils/ping.go

@@ -17,13 +17,24 @@ package testutils
 import (
 	"bytes"
 	"fmt"
+	"net"
 	"os/exec"
 	"strconv"
 	"syscall"
 )
 
 // Ping shells out to the `ping` command. Returns nil if successful.
-func Ping(saddr, daddr string, isV6 bool, timeoutSec int) error {
+func Ping(saddr, daddr string, timeoutSec int) error {
+	ip := net.ParseIP(saddr)
+	if ip == nil {
+		return fmt.Errorf("failed to parse IP %q", saddr)
+	}
+
+	bin := "ping6"
+	if ip.To4() != nil {
+		bin = "ping"
+	}
+
 	args := []string{
 		"-c", "1",
 		"-W", strconv.Itoa(timeoutSec),
@@ -31,11 +42,6 @@ func Ping(saddr, daddr string, isV6 bool, timeoutSec int) error {
 		daddr,
 	}
 
-	bin := "ping"
-	if isV6 {
-		bin = "ping6"
-	}
-
 	cmd := exec.Command(bin, args...)
 	var stderr bytes.Buffer
 	cmd.Stderr = &stderr

pkg/testutils/testing.go

@@ -0,0 +1,54 @@
+// Copyright 2016 CNI authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package testutils
+
+import (
+	"github.com/containernetworking/cni/pkg/version"
+)
+
+// AllSpecVersions contains all CNI spec version numbers
+var AllSpecVersions = [...]string{"0.1.0", "0.2.0", "0.3.0", "0.3.1", "0.4.0", "1.0.0"}
+
+// SpecVersionHasIPVersion returns true if the given CNI specification version
+// includes the "version" field in the IP address elements
+func SpecVersionHasIPVersion(ver string) bool {
+	for _, i := range []string{"0.3.0", "0.3.1", "0.4.0"} {
+		if ver == i {
+			return true
+		}
+	}
+	return false
+}
+
+// SpecVersionHasCHECK returns true if the given CNI specification version
+// supports the CHECK command
+func SpecVersionHasCHECK(ver string) bool {
+	ok, _ := version.GreaterThanOrEqualTo(ver, "0.4.0")
+	return ok
+}
+
+// SpecVersionHasChaining returns true if the given CNI specification version
+// supports plugin chaining
+func SpecVersionHasChaining(ver string) bool {
+	ok, _ := version.GreaterThanOrEqualTo(ver, "0.3.0")
+	return ok
+}
+
+// SpecVersionHasMultipleIPs returns true if the given CNI specification version
+// supports more than one IP address of each family
+func SpecVersionHasMultipleIPs(ver string) bool {
+	ok, _ := version.GreaterThanOrEqualTo(ver, "0.3.0")
+	return ok
+}

pkg/utils/conntrack.go

@@ -62,8 +62,8 @@ func DeleteConntrackEntriesForDstIP(dstIP string, protocol uint8) error {
 // by the given destination port, protocol and IP family
 func DeleteConntrackEntriesForDstPort(port uint16, protocol uint8, family netlink.InetFamily) error {
 	filter := &netlink.ConntrackFilter{}
-	filter.AddPort(netlink.ConntrackOrigDstPort, port)
 	filter.AddProtocol(protocol)
+	filter.AddPort(netlink.ConntrackOrigDstPort, port)
 
 	_, err := netlink.ConntrackDeleteFilter(netlink.ConntrackTable, family, filter)
 	if err != nil {

pkg/utils/hwaddr/hwaddr.go

@@ -1,63 +0,0 @@
-// Copyright 2016 CNI authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package hwaddr
-
-import (
-	"fmt"
-	"net"
-)
-
-const (
-	ipRelevantByteLen      = 4
-	PrivateMACPrefixString = "0a:58"
-)
-
-var (
-	// private mac prefix safe to use
-	PrivateMACPrefix = []byte{0x0a, 0x58}
-)
-
-type SupportIp4OnlyErr struct{ msg string }
-
-func (e SupportIp4OnlyErr) Error() string { return e.msg }
-
-type MacParseErr struct{ msg string }
-
-func (e MacParseErr) Error() string { return e.msg }
-
-type InvalidPrefixLengthErr struct{ msg string }
-
-func (e InvalidPrefixLengthErr) Error() string { return e.msg }
-
-// GenerateHardwareAddr4 generates 48 bit virtual mac addresses based on the IP4 input.
-func GenerateHardwareAddr4(ip net.IP, prefix []byte) (net.HardwareAddr, error) {
-	switch {
-
-	case ip.To4() == nil:
-		return nil, SupportIp4OnlyErr{msg: "GenerateHardwareAddr4 only supports valid IPv4 address as input"}
-
-	case len(prefix) != len(PrivateMACPrefix):
-		return nil, InvalidPrefixLengthErr{msg: fmt.Sprintf(
-			"Prefix has length %d instead  of %d", len(prefix), len(PrivateMACPrefix)),
-		}
-	}
-
-	ipByteLen := len(ip)
-	return (net.HardwareAddr)(
-		append(
-			prefix,
-			ip[ipByteLen-ipRelevantByteLen:ipByteLen]...),
-	), nil
-}

pkg/utils/hwaddr/hwaddr_test.go

@@ -1,74 +0,0 @@
-// Copyright 2016 CNI authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package hwaddr_test
-
-import (
-	"net"
-
-	"github.com/containernetworking/plugins/pkg/utils/hwaddr"
-
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-)
-
-var _ = Describe("Hwaddr", func() {
-	Context("Generate Hardware Address", func() {
-		It("generate hardware address based on ipv4 address", func() {
-			testCases := []struct {
-				ip          net.IP
-				expectedMAC net.HardwareAddr
-			}{
-				{
-					ip:          net.ParseIP("10.0.0.2"),
-					expectedMAC: (net.HardwareAddr)(append(hwaddr.PrivateMACPrefix, 0x0a, 0x00, 0x00, 0x02)),
-				},
-				{
-					ip:          net.ParseIP("10.250.0.244"),
-					expectedMAC: (net.HardwareAddr)(append(hwaddr.PrivateMACPrefix, 0x0a, 0xfa, 0x00, 0xf4)),
-				},
-				{
-					ip:          net.ParseIP("172.17.0.2"),
-					expectedMAC: (net.HardwareAddr)(append(hwaddr.PrivateMACPrefix, 0xac, 0x11, 0x00, 0x02)),
-				},
-				{
-					ip:          net.IPv4(byte(172), byte(17), byte(0), byte(2)),
-					expectedMAC: (net.HardwareAddr)(append(hwaddr.PrivateMACPrefix, 0xac, 0x11, 0x00, 0x02)),
-				},
-			}
-
-			for _, tc := range testCases {
-				mac, err := hwaddr.GenerateHardwareAddr4(tc.ip, hwaddr.PrivateMACPrefix)
-				Expect(err).NotTo(HaveOccurred())
-				Expect(mac).To(Equal(tc.expectedMAC))
-			}
-		})
-
-		It("return error if input is not ipv4 address", func() {
-			testCases := []net.IP{
-				net.ParseIP(""),
-				net.ParseIP("2001:db8:0:1:1:1:1:1"),
-			}
-			for _, tc := range testCases {
-				_, err := hwaddr.GenerateHardwareAddr4(tc, hwaddr.PrivateMACPrefix)
-				Expect(err).To(BeAssignableToTypeOf(hwaddr.SupportIp4OnlyErr{}))
-			}
-		})
-
-		It("return error if prefix is invalid", func() {
-			_, err := hwaddr.GenerateHardwareAddr4(net.ParseIP("10.0.0.2"), []byte{0x58})
-			Expect(err).To(BeAssignableToTypeOf(hwaddr.InvalidPrefixLengthErr{}))
-		})
-	})
-})

pkg/utils/iptables.go

@@ -119,3 +119,21 @@ func ClearChain(ipt *iptables.IPTables, table, chain string) error {
 		return err
 	}
 }
+
+// InsertUnique will add a rule to a chain if it does not already exist.
+// By default the rule is appended, unless prepend is true.
+func InsertUnique(ipt *iptables.IPTables, table, chain string, prepend bool, rule []string) error {
+	exists, err := ipt.Exists(table, chain, rule...)
+	if err != nil {
+		return err
+	}
+	if exists {
+		return nil
+	}
+
+	if prepend {
+		return ipt.Insert(table, chain, 1, rule...)
+	} else {
+		return ipt.Append(table, chain, rule...)
+	}
+}

pkg/utils/sysctl/sysctl_linux.go

@@ -16,7 +16,7 @@ package sysctl
 
 import (
 	"fmt"
-	"io/ioutil"
+	"os"
 	"path/filepath"
 	"strings"
 )
@@ -36,8 +36,7 @@ func Sysctl(name string, params ...string) (string, error) {
 
 func getSysctl(name string) (string, error) {
 	fullName := filepath.Join("/proc/sys", toNormalName(name))
-	fullName = filepath.Clean(fullName)
-	data, err := ioutil.ReadFile(fullName)
+	data, err := os.ReadFile(fullName)
 	if err != nil {
 		return "", err
 	}
@@ -47,8 +46,7 @@ func getSysctl(name string) (string, error) {
 
 func setSysctl(name, value string) (string, error) {
 	fullName := filepath.Join("/proc/sys", toNormalName(name))
-	fullName = filepath.Clean(fullName)
-	if err := ioutil.WriteFile(fullName, []byte(value), 0644); err != nil {
+	if err := os.WriteFile(fullName, []byte(value), 0644); err != nil {
 		return "", err
 	}
 

plugins/ipam/dhcp/README.md

@@ -1,4 +1,4 @@
 
 This document has moved to the [containernetworking/cni.dev](https://github.com/containernetworking/cni.dev) repo.
 
-You can find it online here: https://cni.dev/plugins/ipam/dhcp/
+You can find it online here: https://cni.dev/plugins/current/ipam/dhcp/

plugins/ipam/dhcp/daemon.go

@@ -15,27 +15,26 @@
 package main
 
 import (
+	"context"
 	"encoding/json"
 	"errors"
 	"fmt"
-	"io/ioutil"
 	"net"
 	"net/http"
 	"net/rpc"
 	"os"
+	"os/signal"
 	"path/filepath"
 	"runtime"
 	"sync"
+	"syscall"
 	"time"
 
 	"github.com/containernetworking/cni/pkg/skel"
-	"github.com/containernetworking/cni/pkg/types"
-	"github.com/containernetworking/cni/pkg/types/current"
-	"github.com/coreos/go-systemd/activation"
+	current "github.com/containernetworking/cni/pkg/types/100"
+	"github.com/coreos/go-systemd/v22/activation"
 )
 
-const listenFdsStart = 3
-
 var errNoMoreTries = errors.New("no more tries")
 
 type DHCP struct {
@@ -43,35 +42,59 @@ type DHCP struct {
 	leases          map[string]*DHCPLease
 	hostNetnsPrefix string
 	clientTimeout   time.Duration
+	clientResendMax time.Duration
 	broadcast       bool
 }
 
-func newDHCP(clientTimeout time.Duration) *DHCP {
+func newDHCP(clientTimeout, clientResendMax time.Duration) *DHCP {
 	return &DHCP{
-		leases:        make(map[string]*DHCPLease),
-		clientTimeout: clientTimeout,
+		leases:          make(map[string]*DHCPLease),
+		clientTimeout:   clientTimeout,
+		clientResendMax: clientResendMax,
 	}
 }
 
+// TODO: current client ID is too long. At least the container ID should not be used directly.
+// A seperate issue is necessary to ensure no breaking change is affecting other users.
 func generateClientID(containerID string, netName string, ifName string) string {
-	return containerID + "/" + netName + "/" + ifName
+	clientID := containerID + "/" + netName + "/" + ifName
+	// defined in RFC 2132, length size can not be larger than 1 octet. So we truncate 254 to make everyone happy.
+	if len(clientID) > 254 {
+		clientID = clientID[0:254]
+	}
+	return clientID
 }
 
 // Allocate acquires an IP from a DHCP server for a specified container.
 // The acquired lease will be maintained until Release() is called.
 func (d *DHCP) Allocate(args *skel.CmdArgs, result *current.Result) error {
-	conf := types.NetConf{}
+	conf := NetConf{}
 	if err := json.Unmarshal(args.StdinData, &conf); err != nil {
 		return fmt.Errorf("error parsing netconf: %v", err)
 	}
 
-	clientID := generateClientID(args.ContainerID, conf.Name, args.IfName)
-	hostNetns := d.hostNetnsPrefix + args.Netns
-	l, err := AcquireLease(clientID, hostNetns, args.IfName, d.clientTimeout, d.broadcast)
+	optsRequesting, optsProviding, err := prepareOptions(args.Args, conf.IPAM.ProvideOptions, conf.IPAM.RequestOptions)
 	if err != nil {
 		return err
 	}
 
+	clientID := generateClientID(args.ContainerID, conf.Name, args.IfName)
+
+	// If we already have an active lease for this clientID, do not create
+	// another one
+	l := d.getLease(clientID)
+	if l != nil {
+		l.Check()
+	} else {
+		hostNetns := d.hostNetnsPrefix + args.Netns
+		l, err = AcquireLease(clientID, hostNetns, args.IfName,
+			optsRequesting, optsProviding,
+			d.clientTimeout, d.clientResendMax, d.broadcast)
+		if err != nil {
+			return err
+		}
+	}
+
 	ipn, err := l.IPNet()
 	if err != nil {
 		l.Stop()
@@ -81,7 +104,6 @@ func (d *DHCP) Allocate(args *skel.CmdArgs, result *current.Result) error {
 	d.setLease(clientID, l)
 
 	result.IPs = []*current.IPConfig{{
-		Version: "4",
 		Address: *ipn,
 		Gateway: l.Gateway(),
 	}}
@@ -93,7 +115,7 @@ func (d *DHCP) Allocate(args *skel.CmdArgs, result *current.Result) error {
 // Release stops maintenance of the lease acquired in Allocate()
 // and sends a release msg to the DHCP server.
 func (d *DHCP) Release(args *skel.CmdArgs, reply *struct{}) error {
-	conf := types.NetConf{}
+	conf := NetConf{}
 	if err := json.Unmarshal(args.StdinData, &conf); err != nil {
 		return fmt.Errorf("error parsing netconf: %v", err)
 	}
@@ -127,7 +149,7 @@ func (d *DHCP) setLease(clientID string, l *DHCPLease) {
 	d.leases[clientID] = l
 }
 
-//func (d *DHCP) clearLease(contID, netName, ifName string) {
+// func (d *DHCP) clearLease(contID, netName, ifName string) {
 func (d *DHCP) clearLease(clientID string) {
 	d.mux.Lock()
 	defer d.mux.Unlock()
@@ -162,7 +184,7 @@ func getListener(socketPath string) (net.Listener, error) {
 
 func runDaemon(
 	pidfilePath, hostPrefix, socketPath string,
-	dhcpClientTimeout time.Duration, broadcast bool,
+	dhcpClientTimeout time.Duration, resendMax time.Duration, broadcast bool,
 ) error {
 	// since other goroutines (on separate threads) will change namespaces,
 	// ensure the RPC server does not get scheduled onto those
@@ -173,7 +195,7 @@ func runDaemon(
 		if !filepath.IsAbs(pidfilePath) {
 			return fmt.Errorf("Error writing pidfile %q: path not absolute", pidfilePath)
 		}
-		if err := ioutil.WriteFile(pidfilePath, []byte(fmt.Sprintf("%d", os.Getpid())), 0644); err != nil {
+		if err := os.WriteFile(pidfilePath, []byte(fmt.Sprintf("%d", os.Getpid())), 0644); err != nil {
 			return fmt.Errorf("Error writing pidfile %q: %v", pidfilePath, err)
 		}
 	}
@@ -183,11 +205,27 @@ func runDaemon(
 		return fmt.Errorf("Error getting listener: %v", err)
 	}
 
-	dhcp := newDHCP(dhcpClientTimeout)
+	srv := http.Server{}
+	exit := make(chan os.Signal, 1)
+	done := make(chan bool, 1)
+	signal.Notify(exit, os.Interrupt, syscall.SIGTERM)
+
+	go func() {
+		<-exit
+		srv.Shutdown(context.TODO())
+		os.Remove(hostPrefix + socketPath)
+		os.Remove(pidfilePath)
+
+		done <- true
+	}()
+
+	dhcp := newDHCP(dhcpClientTimeout, resendMax)
 	dhcp.hostNetnsPrefix = hostPrefix
 	dhcp.broadcast = broadcast
 	rpc.Register(dhcp)
 	rpc.HandleHTTP()
-	http.Serve(l, nil)
+	srv.Serve(l)
+
+	<-done
 	return nil
 }

plugins/ipam/dhcp/dhcp2_test.go

@@ -23,7 +23,7 @@ import (
 	"time"
 
 	"github.com/containernetworking/cni/pkg/skel"
-	"github.com/containernetworking/cni/pkg/types/current"
+	current "github.com/containernetworking/cni/pkg/types/100"
 	"github.com/containernetworking/plugins/pkg/ns"
 	"github.com/containernetworking/plugins/pkg/testutils"
 

plugins/ipam/dhcp/dhcp_test.go

@@ -15,8 +15,9 @@
 package main
 
 import (
+	"bytes"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"net"
 	"os"
 	"os/exec"
@@ -25,7 +26,7 @@ import (
 	"time"
 
 	"github.com/containernetworking/cni/pkg/skel"
-	"github.com/containernetworking/cni/pkg/types/current"
+	types100 "github.com/containernetworking/cni/pkg/types/100"
 	"github.com/containernetworking/plugins/pkg/ns"
 	"github.com/containernetworking/plugins/pkg/testutils"
 
@@ -41,7 +42,7 @@ import (
 )
 
 func getTmpDir() (string, error) {
-	tmpDir, err := ioutil.TempDir(cniDirPrefix, "dhcp")
+	tmpDir, err := os.MkdirTemp(cniDirPrefix, "dhcp")
 	if err == nil {
 		tmpDir = filepath.ToSlash(tmpDir)
 	}
@@ -208,6 +209,11 @@ var _ = Describe("DHCP Operations", func() {
 		dhcpPluginPath, err := exec.LookPath("dhcp")
 		Expect(err).NotTo(HaveOccurred())
 		clientCmd = exec.Command(dhcpPluginPath, "daemon", "-socketpath", socketPath)
+
+		// copy dhcp client's stdout/stderr to test stdout
+		clientCmd.Stdout = os.Stdout
+		clientCmd.Stderr = os.Stderr
+
 		err = clientCmd.Start()
 		Expect(err).NotTo(HaveOccurred())
 		Expect(clientCmd.Process).NotTo(BeNil())
@@ -226,118 +232,127 @@ var _ = Describe("DHCP Operations", func() {
 		clientCmd.Wait()
 
 		Expect(originalNS.Close()).To(Succeed())
+		Expect(testutils.UnmountNS(originalNS)).To(Succeed())
 		Expect(targetNS.Close()).To(Succeed())
-		defer os.RemoveAll(tmpDir)
+		Expect(testutils.UnmountNS(targetNS)).To(Succeed())
+
+		Expect(os.RemoveAll(tmpDir)).To(Succeed())
 	})
 
-	It("configures and deconfigures a link with ADD/DEL", func() {
-		conf := fmt.Sprintf(`{
-    "cniVersion": "0.3.1",
-    "name": "mynet",
-    "type": "ipvlan",
-    "ipam": {
-        "type": "dhcp",
-	"daemonSocketPath": "%s"
-    }
-}`, socketPath)
+	for _, ver := range testutils.AllSpecVersions {
+		// Redefine ver inside for scope so real value is picked up by each dynamically defined It()
+		// See Gingkgo's "Patterns for dynamically generating tests" documentation.
+		ver := ver
 
-		args := &skel.CmdArgs{
-			ContainerID: "dummy",
-			Netns:       targetNS.Path(),
-			IfName:      contVethName,
-			StdinData:   []byte(conf),
-		}
+		It(fmt.Sprintf("[%s] configures and deconfigures a link with ADD/DEL", ver), func() {
+			conf := fmt.Sprintf(`{
+			    "cniVersion": "%s",
+			    "name": "mynet",
+			    "type": "ipvlan",
+			    "ipam": {
+				"type": "dhcp",
+				"daemonSocketPath": "%s"
+			    }
+			}`, ver, socketPath)
 
-		var addResult *current.Result
-		err := originalNS.Do(func(ns.NetNS) error {
-			defer GinkgoRecover()
+			args := &skel.CmdArgs{
+				ContainerID: "dummy",
+				Netns:       targetNS.Path(),
+				IfName:      contVethName,
+				StdinData:   []byte(conf),
+			}
 
-			r, _, err := testutils.CmdAddWithArgs(args, func() error {
-				return cmdAdd(args)
-			})
-			Expect(err).NotTo(HaveOccurred())
-
-			addResult, err = current.GetResult(r)
-			Expect(err).NotTo(HaveOccurred())
-			Expect(len(addResult.IPs)).To(Equal(1))
-			Expect(addResult.IPs[0].Address.String()).To(Equal("192.168.1.5/24"))
-			return nil
-		})
-		Expect(err).NotTo(HaveOccurred())
-
-		err = originalNS.Do(func(ns.NetNS) error {
-			return testutils.CmdDelWithArgs(args, func() error {
-				return cmdDel(args)
-			})
-		})
-		Expect(err).NotTo(HaveOccurred())
-	})
-
-	It("correctly handles multiple DELs for the same container", func() {
-		conf := fmt.Sprintf(`{
-    "cniVersion": "0.3.1",
-    "name": "mynet",
-    "type": "ipvlan",
-    "ipam": {
-        "type": "dhcp",
-	"daemonSocketPath": "%s"
-    }
-}`, socketPath)
-
-		args := &skel.CmdArgs{
-			ContainerID: "dummy",
-			Netns:       targetNS.Path(),
-			IfName:      contVethName,
-			StdinData:   []byte(conf),
-		}
-
-		var addResult *current.Result
-		err := originalNS.Do(func(ns.NetNS) error {
-			defer GinkgoRecover()
-
-			r, _, err := testutils.CmdAddWithArgs(args, func() error {
-				return cmdAdd(args)
-			})
-			Expect(err).NotTo(HaveOccurred())
-
-			addResult, err = current.GetResult(r)
-			Expect(err).NotTo(HaveOccurred())
-			Expect(len(addResult.IPs)).To(Equal(1))
-			Expect(addResult.IPs[0].Address.String()).To(Equal("192.168.1.5/24"))
-			return nil
-		})
-		Expect(err).NotTo(HaveOccurred())
-
-		wg := sync.WaitGroup{}
-		wg.Add(3)
-		started := sync.WaitGroup{}
-		started.Add(3)
-		for i := 0; i < 3; i++ {
-			go func() {
+			var addResult *types100.Result
+			err := originalNS.Do(func(ns.NetNS) error {
 				defer GinkgoRecover()
 
-				// Wait until all goroutines are running
-				started.Done()
-				started.Wait()
-
-				err = originalNS.Do(func(ns.NetNS) error {
-					return testutils.CmdDelWithArgs(args, func() error {
-						return cmdDel(args)
-					})
+				r, _, err := testutils.CmdAddWithArgs(args, func() error {
+					return cmdAdd(args)
 				})
 				Expect(err).NotTo(HaveOccurred())
-				wg.Done()
-			}()
-		}
-		wg.Wait()
 
-		err = originalNS.Do(func(ns.NetNS) error {
-			return testutils.CmdDelWithArgs(args, func() error {
-				return cmdDel(args)
+				addResult, err = types100.GetResult(r)
+				Expect(err).NotTo(HaveOccurred())
+				Expect(len(addResult.IPs)).To(Equal(1))
+				Expect(addResult.IPs[0].Address.String()).To(Equal("192.168.1.5/24"))
+				return nil
 			})
+			Expect(err).NotTo(HaveOccurred())
+
+			err = originalNS.Do(func(ns.NetNS) error {
+				return testutils.CmdDelWithArgs(args, func() error {
+					return cmdDel(args)
+				})
+			})
+			Expect(err).NotTo(HaveOccurred())
 		})
-		Expect(err).NotTo(HaveOccurred())
-	})
+
+		It(fmt.Sprintf("[%s] correctly handles multiple DELs for the same container", ver), func() {
+			conf := fmt.Sprintf(`{
+			    "cniVersion": "%s",
+			    "name": "mynet",
+			    "type": "ipvlan",
+			    "ipam": {
+				"type": "dhcp",
+				"daemonSocketPath": "%s"
+			    }
+			}`, ver, socketPath)
+
+			args := &skel.CmdArgs{
+				ContainerID: "dummy",
+				Netns:       targetNS.Path(),
+				IfName:      contVethName,
+				StdinData:   []byte(conf),
+			}
+
+			var addResult *types100.Result
+			err := originalNS.Do(func(ns.NetNS) error {
+				defer GinkgoRecover()
+
+				r, _, err := testutils.CmdAddWithArgs(args, func() error {
+					return cmdAdd(args)
+				})
+				Expect(err).NotTo(HaveOccurred())
+
+				addResult, err = types100.GetResult(r)
+				Expect(err).NotTo(HaveOccurred())
+				Expect(len(addResult.IPs)).To(Equal(1))
+				Expect(addResult.IPs[0].Address.String()).To(Equal("192.168.1.5/24"))
+				return nil
+			})
+			Expect(err).NotTo(HaveOccurred())
+
+			wg := sync.WaitGroup{}
+			wg.Add(3)
+			started := sync.WaitGroup{}
+			started.Add(3)
+			for i := 0; i < 3; i++ {
+				go func() {
+					defer GinkgoRecover()
+
+					// Wait until all goroutines are running
+					started.Done()
+					started.Wait()
+
+					err = originalNS.Do(func(ns.NetNS) error {
+						return testutils.CmdDelWithArgs(args, func() error {
+							return cmdDel(args)
+						})
+					})
+					Expect(err).NotTo(HaveOccurred())
+					wg.Done()
+				}()
+			}
+			wg.Wait()
+
+			err = originalNS.Do(func(ns.NetNS) error {
+				return testutils.CmdDelWithArgs(args, func() error {
+					return cmdDel(args)
+				})
+			})
+			Expect(err).NotTo(HaveOccurred())
+		})
+	}
 })
 
 const (
@@ -508,7 +523,19 @@ var _ = Describe("DHCP Lease Unavailable Operations", func() {
 		// Start the DHCP client daemon
 		dhcpPluginPath, err := exec.LookPath("dhcp")
 		Expect(err).NotTo(HaveOccurred())
-		clientCmd = exec.Command(dhcpPluginPath, "daemon", "-socketpath", socketPath)
+		// Use very short timeouts for lease-unavailable operations because
+		// the same test is run many times, and the delays will exceed the
+		// `go test` timeout with default delays. Since our DHCP server
+		// and client daemon are local processes anyway, we can depend on
+		// them to respond very quickly.
+		clientCmd = exec.Command(dhcpPluginPath, "daemon", "-socketpath", socketPath, "-timeout", "2s", "-resendmax", "8s")
+
+		// copy dhcp client's stdout/stderr to test stdout
+		var b bytes.Buffer
+		mw := io.MultiWriter(os.Stdout, &b)
+		clientCmd.Stdout = mw
+		clientCmd.Stderr = mw
+
 		err = clientCmd.Start()
 		Expect(err).NotTo(HaveOccurred())
 		Expect(clientCmd.Process).NotTo(BeNil())
@@ -527,92 +554,101 @@ var _ = Describe("DHCP Lease Unavailable Operations", func() {
 		clientCmd.Wait()
 
 		Expect(originalNS.Close()).To(Succeed())
+		Expect(testutils.UnmountNS(originalNS)).To(Succeed())
 		Expect(targetNS.Close()).To(Succeed())
-		defer os.RemoveAll(tmpDir)
+		Expect(testutils.UnmountNS(targetNS)).To(Succeed())
+
+		Expect(os.RemoveAll(tmpDir)).To(Succeed())
 	})
 
-	It("Configures multiple links with multiple ADD with second lease unavailable", func() {
-		conf := fmt.Sprintf(`{
-	    "cniVersion": "0.3.1",
-	    "name": "mynet",
-	    "type": "bridge",
-	    "bridge": "%s",
-	    "ipam": {
-	        "type": "dhcp",
-		"daemonSocketPath": "%s"
-	    }
-	}`, hostBridgeName, socketPath)
+	for _, ver := range testutils.AllSpecVersions {
+		// Redefine ver inside for scope so real value is picked up by each dynamically defined It()
+		// See Gingkgo's "Patterns for dynamically generating tests" documentation.
+		ver := ver
 
-		args := &skel.CmdArgs{
-			ContainerID: "dummy",
-			Netns:       targetNS.Path(),
-			IfName:      contVethName0,
-			StdinData:   []byte(conf),
-		}
+		It(fmt.Sprintf("[%s] configures multiple links with multiple ADD with second lease unavailable", ver), func() {
+			conf := fmt.Sprintf(`{
+			    "cniVersion": "%s",
+			    "name": "mynet",
+			    "type": "bridge",
+			    "bridge": "%s",
+			    "ipam": {
+				"type": "dhcp",
+				"daemonSocketPath": "%s"
+			    }
+			}`, ver, hostBridgeName, socketPath)
 
-		var addResult *current.Result
-		err := originalNS.Do(func(ns.NetNS) error {
-			defer GinkgoRecover()
+			args := &skel.CmdArgs{
+				ContainerID: "dummy",
+				Netns:       targetNS.Path(),
+				IfName:      contVethName0,
+				StdinData:   []byte(conf),
+			}
 
-			r, _, err := testutils.CmdAddWithArgs(args, func() error {
-				return cmdAdd(args)
+			var addResult *types100.Result
+			err := originalNS.Do(func(ns.NetNS) error {
+				defer GinkgoRecover()
+
+				r, _, err := testutils.CmdAddWithArgs(args, func() error {
+					return cmdAdd(args)
+				})
+				Expect(err).NotTo(HaveOccurred())
+
+				addResult, err = types100.GetResult(r)
+				Expect(err).NotTo(HaveOccurred())
+				Expect(len(addResult.IPs)).To(Equal(1))
+				Expect(addResult.IPs[0].Address.String()).To(Equal("192.168.1.5/24"))
+				return nil
 			})
 			Expect(err).NotTo(HaveOccurred())
 
-			addResult, err = current.GetResult(r)
+			args = &skel.CmdArgs{
+				ContainerID: "dummy",
+				Netns:       targetNS.Path(),
+				IfName:      contVethName1,
+				StdinData:   []byte(conf),
+			}
+
+			err = originalNS.Do(func(ns.NetNS) error {
+				defer GinkgoRecover()
+
+				_, _, err := testutils.CmdAddWithArgs(args, func() error {
+					return cmdAdd(args)
+				})
+				Expect(err).To(HaveOccurred())
+				println(err.Error())
+				Expect(err.Error()).To(Equal("error calling DHCP.Allocate: no more tries"))
+				return nil
+			})
 			Expect(err).NotTo(HaveOccurred())
-			Expect(len(addResult.IPs)).To(Equal(1))
-			Expect(addResult.IPs[0].Address.String()).To(Equal("192.168.1.5/24"))
-			return nil
-		})
-		Expect(err).NotTo(HaveOccurred())
 
-		args = &skel.CmdArgs{
-			ContainerID: "dummy",
-			Netns:       targetNS.Path(),
-			IfName:      contVethName1,
-			StdinData:   []byte(conf),
-		}
+			args = &skel.CmdArgs{
+				ContainerID: "dummy",
+				Netns:       targetNS.Path(),
+				IfName:      contVethName1,
+				StdinData:   []byte(conf),
+			}
 
-		err = originalNS.Do(func(ns.NetNS) error {
-			defer GinkgoRecover()
-
-			_, _, err := testutils.CmdAddWithArgs(args, func() error {
-				return cmdAdd(args)
+			err = originalNS.Do(func(ns.NetNS) error {
+				return testutils.CmdDelWithArgs(args, func() error {
+					return cmdDel(args)
+				})
 			})
-			Expect(err).To(HaveOccurred())
-			println(err.Error())
-			Expect(err.Error()).To(Equal("error calling DHCP.Allocate: no more tries"))
-			return nil
-		})
-		Expect(err).NotTo(HaveOccurred())
+			Expect(err).NotTo(HaveOccurred())
 
-		args = &skel.CmdArgs{
-			ContainerID: "dummy",
-			Netns:       targetNS.Path(),
-			IfName:      contVethName1,
-			StdinData:   []byte(conf),
-		}
+			args = &skel.CmdArgs{
+				ContainerID: "dummy",
+				Netns:       targetNS.Path(),
+				IfName:      contVethName0,
+				StdinData:   []byte(conf),
+			}
 
-		err = originalNS.Do(func(ns.NetNS) error {
-			return testutils.CmdDelWithArgs(args, func() error {
-				return cmdDel(args)
+			err = originalNS.Do(func(ns.NetNS) error {
+				return testutils.CmdDelWithArgs(args, func() error {
+					return cmdDel(args)
+				})
 			})
+			Expect(err).NotTo(HaveOccurred())
 		})
-		Expect(err).NotTo(HaveOccurred())
-
-		args = &skel.CmdArgs{
-			ContainerID: "dummy",
-			Netns:       targetNS.Path(),
-			IfName:      contVethName0,
-			StdinData:   []byte(conf),
-		}
-
-		err = originalNS.Do(func(ns.NetNS) error {
-			return testutils.CmdDelWithArgs(args, func() error {
-				return cmdDel(args)
-			})
-		})
-		Expect(err).NotTo(HaveOccurred())
-	})
+	}
 })

plugins/ipam/dhcp/lease.go

@@ -19,6 +19,7 @@ import (
 	"log"
 	"math/rand"
 	"net"
+	"strings"
 	"sync"
 	"sync/atomic"
 	"time"
@@ -36,6 +37,11 @@ import (
 const resendDelay0 = 4 * time.Second
 const resendDelayMax = 62 * time.Second
 
+// To speed up the retry for first few failures, we retry without
+// backoff for a few times
+const resendFastDelay = 2 * time.Second
+const resendFastMax = 4
+
 const (
 	leaseStateBound = iota
 	leaseStateRenewing
@@ -57,10 +63,80 @@ type DHCPLease struct {
 	rebindingTime time.Time
 	expireTime    time.Time
 	timeout       time.Duration
+	resendMax     time.Duration
 	broadcast     bool
 	stopping      uint32
 	stop          chan struct{}
+	check         chan struct{}
 	wg            sync.WaitGroup
+	// list of requesting and providing options and if they are necessary / their value
+	optsRequesting map[dhcp4.OptionCode]bool
+	optsProviding  map[dhcp4.OptionCode][]byte
+}
+
+var requestOptionsDefault = map[dhcp4.OptionCode]bool{
+	dhcp4.OptionRouter:     true,
+	dhcp4.OptionSubnetMask: true,
+}
+
+func prepareOptions(cniArgs string, ProvideOptions []ProvideOption, RequestOptions []RequestOption) (
+	optsRequesting map[dhcp4.OptionCode]bool, optsProviding map[dhcp4.OptionCode][]byte, err error) {
+
+	// parse CNI args
+	cniArgsParsed := map[string]string{}
+	for _, argPair := range strings.Split(cniArgs, ";") {
+		args := strings.SplitN(argPair, "=", 2)
+		if len(args) > 1 {
+			cniArgsParsed[args[0]] = args[1]
+		}
+	}
+
+	// parse providing options map
+	var optParsed dhcp4.OptionCode
+	optsProviding = make(map[dhcp4.OptionCode][]byte)
+	for _, opt := range ProvideOptions {
+		optParsed, err = parseOptionName(string(opt.Option))
+		if err != nil {
+			err = fmt.Errorf("Can not parse option %q: %w", opt.Option, err)
+			return
+		}
+		if len(opt.Value) > 0 {
+			if len(opt.Value) > 255 {
+				err = fmt.Errorf("value too long for option %q: %q", opt.Option, opt.Value)
+				return
+			}
+			optsProviding[optParsed] = []byte(opt.Value)
+		}
+		if value, ok := cniArgsParsed[opt.ValueFromCNIArg]; ok {
+			if len(value) > 255 {
+				err = fmt.Errorf("value too long for option %q from CNI_ARGS %q: %q", opt.Option, opt.ValueFromCNIArg, opt.Value)
+				return
+			}
+			optsProviding[optParsed] = []byte(value)
+		}
+	}
+
+	// parse necessary options map
+	optsRequesting = make(map[dhcp4.OptionCode]bool)
+	skipRequireDefault := false
+	for _, opt := range RequestOptions {
+		if opt.SkipDefault {
+			skipRequireDefault = true
+		}
+		optParsed, err = parseOptionName(string(opt.Option))
+		if err != nil {
+			err = fmt.Errorf("Can not parse option %q: %w", opt.Option, err)
+			return
+		}
+		optsRequesting[optParsed] = true
+	}
+	for k, v := range requestOptionsDefault {
+		// only set if not skipping default and this value does not exists
+		if _, ok := optsRequesting[k]; !ok && !skipRequireDefault {
+			optsRequesting[k] = v
+		}
+	}
+	return
 }
 
 // AcquireLease gets an DHCP lease and then maintains it in the background
@@ -68,14 +144,19 @@ type DHCPLease struct {
 // calling DHCPLease.Stop()
 func AcquireLease(
 	clientID, netns, ifName string,
-	timeout time.Duration, broadcast bool,
+	optsRequesting map[dhcp4.OptionCode]bool, optsProviding map[dhcp4.OptionCode][]byte,
+	timeout, resendMax time.Duration, broadcast bool,
 ) (*DHCPLease, error) {
 	errCh := make(chan error, 1)
 	l := &DHCPLease{
-		clientID:  clientID,
-		stop:      make(chan struct{}),
-		timeout:   timeout,
-		broadcast: broadcast,
+		clientID:       clientID,
+		stop:           make(chan struct{}),
+		check:          make(chan struct{}),
+		timeout:        timeout,
+		resendMax:      resendMax,
+		broadcast:      broadcast,
+		optsRequesting: optsRequesting,
+		optsProviding:  optsProviding,
 	}
 
 	log.Printf("%v: acquiring lease", clientID)
@@ -121,6 +202,34 @@ func (l *DHCPLease) Stop() {
 	l.wg.Wait()
 }
 
+func (l *DHCPLease) Check() {
+	l.check <- struct{}{}
+}
+
+func (l *DHCPLease) getOptionsWithClientId() dhcp4.Options {
+	opts := make(dhcp4.Options)
+	opts[dhcp4.OptionClientIdentifier] = []byte(l.clientID)
+	// client identifier's first byte is "type"
+	newClientID := []byte{0}
+	newClientID = append(newClientID, opts[dhcp4.OptionClientIdentifier]...)
+	opts[dhcp4.OptionClientIdentifier] = newClientID
+	return opts
+}
+
+func (l *DHCPLease) getAllOptions() dhcp4.Options {
+	opts := l.getOptionsWithClientId()
+
+	for k, v := range l.optsProviding {
+		opts[k] = v
+	}
+
+	opts[dhcp4.OptionParameterRequestList] = []byte{}
+	for k := range l.optsRequesting {
+		opts[dhcp4.OptionParameterRequestList] = append(opts[dhcp4.OptionParameterRequestList], byte(k))
+	}
+	return opts
+}
+
 func (l *DHCPLease) acquire() error {
 	c, err := newDHCPClient(l.link, l.clientID, l.timeout, l.broadcast)
 	if err != nil {
@@ -135,11 +244,9 @@ func (l *DHCPLease) acquire() error {
 		}
 	}
 
-	opts := make(dhcp4.Options)
-	opts[dhcp4.OptionClientIdentifier] = []byte(l.clientID)
-	opts[dhcp4.OptionParameterRequestList] = []byte{byte(dhcp4.OptionRouter), byte(dhcp4.OptionSubnetMask)}
+	opts := l.getAllOptions()
 
-	pkt, err := backoffRetry(func() (*dhcp4.Packet, error) {
+	pkt, err := backoffRetry(l.resendMax, func() (*dhcp4.Packet, error) {
 		ok, ack, err := DhcpRequest(c, opts)
 		switch {
 		case err != nil:
@@ -207,7 +314,7 @@ func (l *DHCPLease) maintain() {
 				log.Printf("%v: %v", l.clientID, err)
 
 				if time.Now().After(l.rebindingTime) {
-					log.Printf("%v: renawal time expired, rebinding", l.clientID)
+					log.Printf("%v: renewal time expired, rebinding", l.clientID)
 					state = leaseStateRebinding
 				}
 			} else {
@@ -233,6 +340,9 @@ func (l *DHCPLease) maintain() {
 		select {
 		case <-time.After(sleepDur):
 
+		case <-l.check:
+			log.Printf("%v: Checking lease", l.clientID)
+
 		case <-l.stop:
 			if err := l.release(); err != nil {
 				log.Printf("%v: failed to release DHCP lease: %v", l.clientID, err)
@@ -255,10 +365,8 @@ func (l *DHCPLease) renew() error {
 	}
 	defer c.Close()
 
-	opts := make(dhcp4.Options)
-	opts[dhcp4.OptionClientIdentifier] = []byte(l.clientID)
-
-	pkt, err := backoffRetry(func() (*dhcp4.Packet, error) {
+	opts := l.getAllOptions()
+	pkt, err := backoffRetry(l.resendMax, func() (*dhcp4.Packet, error) {
 		ok, ack, err := DhcpRenew(c, *l.ack, opts)
 		switch {
 		case err != nil:
@@ -286,8 +394,7 @@ func (l *DHCPLease) release() error {
 	}
 	defer c.Close()
 
-	opts := make(dhcp4.Options)
-	opts[dhcp4.OptionClientIdentifier] = []byte(l.clientID)
+	opts := l.getOptionsWithClientId()
 
 	if err = DhcpRelease(c, *l.ack, opts); err != nil {
 		return fmt.Errorf("failed to send DHCPRELEASE")
@@ -340,10 +447,10 @@ func jitter(span time.Duration) time.Duration {
 	return time.Duration(float64(span) * (2.0*rand.Float64() - 1.0))
 }
 
-func backoffRetry(f func() (*dhcp4.Packet, error)) (*dhcp4.Packet, error) {
+func backoffRetry(resendMax time.Duration, f func() (*dhcp4.Packet, error)) (*dhcp4.Packet, error) {
 	var baseDelay time.Duration = resendDelay0
 	var sleepTime time.Duration
-
+	var fastRetryLimit = resendFastMax
 	for {
 		pkt, err := f()
 		if err == nil {
@@ -352,15 +459,21 @@ func backoffRetry(f func() (*dhcp4.Packet, error)) (*dhcp4.Packet, error) {
 
 		log.Print(err)
 
-		sleepTime = baseDelay + jitter(time.Second)
+		if fastRetryLimit == 0 {
+			sleepTime = baseDelay + jitter(time.Second)
+		} else {
+			sleepTime = resendFastDelay + jitter(time.Second)
+			fastRetryLimit--
+		}
 
 		log.Printf("retrying in %f seconds", sleepTime.Seconds())
 
 		time.Sleep(sleepTime)
 
-		if baseDelay < resendDelayMax {
+		// only adjust delay time if we are in normal backoff stage
+		if baseDelay < resendMax && fastRetryLimit == 0 {
 			baseDelay *= 2
-		} else {
+		} else if fastRetryLimit == 0 { // only break if we are at normal delay
 			break
 		}
 	}

plugins/ipam/dhcp/main.go

@@ -26,13 +26,50 @@ import (
 
 	"github.com/containernetworking/cni/pkg/skel"
 	"github.com/containernetworking/cni/pkg/types"
-	"github.com/containernetworking/cni/pkg/types/current"
+	current "github.com/containernetworking/cni/pkg/types/100"
 	"github.com/containernetworking/cni/pkg/version"
 	bv "github.com/containernetworking/plugins/pkg/utils/buildversion"
 )
 
 const defaultSocketPath = "/run/cni/dhcp.sock"
 
+// The top-level network config - IPAM plugins are passed the full configuration
+// of the calling plugin, not just the IPAM section.
+type NetConf struct {
+	types.NetConf
+	IPAM *IPAMConfig `json:"ipam"`
+}
+
+type IPAMConfig struct {
+	types.IPAM
+	DaemonSocketPath string `json:"daemonSocketPath"`
+	// When requesting IP from DHCP server, carry these options for management purpose.
+	// Some fields have default values, and can be override by setting a new option with the same name at here.
+	ProvideOptions []ProvideOption `json:"provide"`
+	// When requesting IP from DHCP server, claiming these options are necessary. Options are necessary unless `optional`
+	// is set to `false`.
+	// To override default requesting fields, set `skipDefault` to `false`.
+	// If an field is not optional, but the server failed to provide it, error will be raised.
+	RequestOptions []RequestOption `json:"request"`
+}
+
+// DHCPOption represents a DHCP option. It can be a number, or a string defined in manual dhcp-options(5).
+// Note that not all DHCP options are supported at all time. Error will be raised if unsupported options are used.
+type DHCPOption string
+
+type ProvideOption struct {
+	Option DHCPOption `json:"option"`
+
+	Value           string `json:"value"`
+	ValueFromCNIArg string `json:"fromArg"`
+}
+
+type RequestOption struct {
+	SkipDefault bool `json:"skipDefault"`
+
+	Option DHCPOption `json:"option"`
+}
+
 func main() {
 	if len(os.Args) > 1 && os.Args[1] == "daemon" {
 		var pidfilePath string
@@ -40,20 +77,22 @@ func main() {
 		var socketPath string
 		var broadcast bool
 		var timeout time.Duration
+		var resendMax time.Duration
 		daemonFlags := flag.NewFlagSet("daemon", flag.ExitOnError)
 		daemonFlags.StringVar(&pidfilePath, "pidfile", "", "optional path to write daemon PID to")
 		daemonFlags.StringVar(&hostPrefix, "hostprefix", "", "optional prefix to host root")
 		daemonFlags.StringVar(&socketPath, "socketpath", "", "optional dhcp server socketpath")
 		daemonFlags.BoolVar(&broadcast, "broadcast", false, "broadcast DHCP leases")
 		daemonFlags.DurationVar(&timeout, "timeout", 10*time.Second, "optional dhcp client timeout duration")
+		daemonFlags.DurationVar(&resendMax, "resendmax", resendDelayMax, "optional dhcp client resend max duration")
 		daemonFlags.Parse(os.Args[2:])
 
 		if socketPath == "" {
 			socketPath = defaultSocketPath
 		}
 
-		if err := runDaemon(pidfilePath, hostPrefix, socketPath, timeout, broadcast); err != nil {
-			log.Printf(err.Error())
+		if err := runDaemon(pidfilePath, hostPrefix, socketPath, timeout, resendMax, broadcast); err != nil {
+			log.Print(err.Error())
 			os.Exit(1)
 		}
 	} else {
@@ -69,7 +108,7 @@ func cmdAdd(args *skel.CmdArgs) error {
 		return err
 	}
 
-	result := &current.Result{}
+	result := &current.Result{CNIVersion: current.ImplementedSpecVersion}
 	if err := rpcCall("DHCP.Allocate", args, result); err != nil {
 		return err
 	}
@@ -86,8 +125,6 @@ func cmdDel(args *skel.CmdArgs) error {
 }
 
 func cmdCheck(args *skel.CmdArgs) error {
-	// TODO: implement
-	//return fmt.Errorf("not implemented")
 	// Plugin must return result in same version as specified in netconf
 	versionDecoder := &version.ConfigDecoder{}
 	//confVersion, err := versionDecoder.Decode(args.StdinData)
@@ -96,7 +133,7 @@ func cmdCheck(args *skel.CmdArgs) error {
 		return err
 	}
 
-	result := &current.Result{}
+	result := &current.Result{CNIVersion: current.ImplementedSpecVersion}
 	if err := rpcCall("DHCP.Allocate", args, result); err != nil {
 		return err
 	}
@@ -104,16 +141,8 @@ func cmdCheck(args *skel.CmdArgs) error {
 	return nil
 }
 
-type SocketPathConf struct {
-	DaemonSocketPath string `json:"daemonSocketPath,omitempty"`
-}
-
-type TempNetConf struct {
-	IPAM SocketPathConf `json:"ipam,omitempty"`
-}
-
 func getSocketPath(stdinData []byte) (string, error) {
-	conf := TempNetConf{}
+	conf := NetConf{}
 	if err := json.Unmarshal(stdinData, &conf); err != nil {
 		return "", fmt.Errorf("error parsing socket path conf: %v", err)
 	}

plugins/ipam/dhcp/options.go

@@ -18,12 +18,33 @@ import (
 	"encoding/binary"
 	"fmt"
 	"net"
+	"strconv"
 	"time"
 
 	"github.com/containernetworking/cni/pkg/types"
 	"github.com/d2g/dhcp4"
 )
 
+var optionNameToID = map[string]dhcp4.OptionCode{
+	"dhcp-client-identifier":  dhcp4.OptionClientIdentifier,
+	"subnet-mask":             dhcp4.OptionSubnetMask,
+	"routers":                 dhcp4.OptionRouter,
+	"host-name":               dhcp4.OptionHostName,
+	"user-class":              dhcp4.OptionUserClass,
+	"vendor-class-identifier": dhcp4.OptionVendorClassIdentifier,
+}
+
+func parseOptionName(option string) (dhcp4.OptionCode, error) {
+	if val, ok := optionNameToID[option]; ok {
+		return val, nil
+	}
+	i, err := strconv.ParseUint(option, 10, 8)
+	if err != nil {
+		return 0, fmt.Errorf("Can not parse option: %w", err)
+	}
+	return dhcp4.OptionCode(i), nil
+}
+
 func parseRouter(opts dhcp4.Options) net.IP {
 	if opts, ok := opts[dhcp4.OptionRouter]; ok {
 		if len(opts) == 4 {

plugins/ipam/dhcp/options_test.go

@@ -16,6 +16,7 @@ package main
 
 import (
 	"net"
+	"reflect"
 	"testing"
 
 	"github.com/containernetworking/cni/pkg/types"
@@ -73,3 +74,34 @@ func TestParseCIDRRoutes(t *testing.T) {
 
 	validateRoutes(t, routes)
 }
+
+func TestParseOptionName(t *testing.T) {
+	tests := []struct {
+		name    string
+		option  string
+		want    dhcp4.OptionCode
+		wantErr bool
+	}{
+		{
+			"hostname", "host-name", dhcp4.OptionHostName, false,
+		},
+		{
+			"hostname in number", "12", dhcp4.OptionHostName, false,
+		},
+		{
+			"random string", "doNotparseMe", 0, true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := parseOptionName(tt.option)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("parseOptionName() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("parseOptionName() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

plugins/ipam/host-local/README.md

@@ -1,4 +1,4 @@
 
 This document has moved to the [containernetworking/cni.dev](https://github.com/containernetworking/cni.dev) repo.
 
-You can find it online here: https://cni.dev/plugins/ipam/host-local/
+You can find it online here: https://cni.dev/plugins/current/ipam/host-local/

plugins/ipam/host-local/backend/allocator/allocator.go

@@ -21,7 +21,8 @@ import (
 	"os"
 	"strconv"
 
-	"github.com/containernetworking/cni/pkg/types/current"
+	current "github.com/containernetworking/cni/pkg/types/100"
+
 	"github.com/containernetworking/plugins/pkg/ip"
 	"github.com/containernetworking/plugins/plugins/ipam/host-local/backend"
 )
@@ -108,13 +109,8 @@ func (a *IPAllocator) Get(id string, ifname string, requestedIP net.IP) (*curren
 	if reservedIP == nil {
 		return nil, fmt.Errorf("no IP addresses available in range set: %s", a.rangeset.String())
 	}
-	version := "4"
-	if reservedIP.IP.To4() == nil {
-		version = "6"
-	}
 
 	return &current.IPConfig{
-		Version: version,
 		Address: *reservedIP,
 		Gateway: gw,
 	}, nil
@@ -137,9 +133,8 @@ type RangeIter struct {
 	// Our current position
 	cur net.IP
 
-	// The IP and range index where we started iterating; if we hit this again, we're done.
-	startIP    net.IP
-	startRange int
+	// The IP where we started iterating; if we hit this again, we're done.
+	startIP net.IP
 }
 
 // GetIter encapsulates the strategy for this allocator.
@@ -169,7 +164,6 @@ func (a *IPAllocator) GetIter() (*RangeIter, error) {
 		for i, r := range *a.rangeset {
 			if r.Contains(lastReservedIP) {
 				iter.rangeIdx = i
-				iter.startRange = i
 
 				// We advance the cursor on every Next(), so the first call
 				// to next() will return lastReservedIP + 1
@@ -179,7 +173,6 @@ func (a *IPAllocator) GetIter() (*RangeIter, error) {
 		}
 	} else {
 		iter.rangeIdx = 0
-		iter.startRange = 0
 		iter.startIP = (*a.rangeset)[0].RangeStart
 	}
 	return &iter, nil
@@ -215,7 +208,7 @@ func (i *RangeIter) Next() (*net.IPNet, net.IP) {
 
 	if i.startIP == nil {
 		i.startIP = i.cur
-	} else if i.rangeIdx == i.startRange && i.cur.Equal(i.startIP) {
+	} else if i.cur.Equal(i.startIP) {
 		// IF we've looped back to where we started, give up
 		return nil, nil
 	}

plugins/ipam/host-local/backend/allocator/allocator_test.go

@@ -19,7 +19,7 @@ import (
 	"net"
 
 	"github.com/containernetworking/cni/pkg/types"
-	"github.com/containernetworking/cni/pkg/types/current"
+	current "github.com/containernetworking/cni/pkg/types/100"
 	fakestore "github.com/containernetworking/plugins/plugins/ipam/host-local/backend/testing"
 
 	. "github.com/onsi/ginkgo"
@@ -49,6 +49,23 @@ func mkalloc() IPAllocator {
 	return alloc
 }
 
+func newAllocatorWithMultiRanges() IPAllocator {
+	p := RangeSet{
+		Range{RangeStart: net.IP{192, 168, 1, 0}, RangeEnd: net.IP{192, 168, 1, 3}, Subnet: mustSubnet("192.168.1.1/30")},
+		Range{RangeStart: net.IP{192, 168, 2, 0}, RangeEnd: net.IP{192, 168, 2, 3}, Subnet: mustSubnet("192.168.2.1/30")},
+	}
+	_ = p.Canonicalize()
+	store := fakestore.NewFakeStore(map[string]string{}, map[string]net.IP{})
+
+	alloc := IPAllocator{
+		rangeset: &p,
+		store:    store,
+		rangeID:  "rangeid",
+	}
+
+	return alloc
+}
+
 func (t AllocatorTestCase) run(idx int) (*current.IPConfig, error) {
 	fmt.Fprintln(GinkgoWriter, "Index:", idx)
 	p := RangeSet{}
@@ -320,6 +337,39 @@ var _ = Describe("host-local ip allocator", func() {
 			}
 		})
 	})
+
+	Context("when lastReservedIP is at the end of one of multi ranges", func() {
+		It("should use the first IP of next range as startIP after Next", func() {
+			a := newAllocatorWithMultiRanges()
+
+			// reserve the last IP of the first range
+			reserved, err := a.store.Reserve("ID", "eth0", net.IP{192, 168, 1, 3}, a.rangeID)
+			Expect(reserved).To(BeTrue())
+			Expect(err).NotTo(HaveOccurred())
+
+			// get range iterator and do the first Next
+			r, err := a.GetIter()
+			Expect(err).NotTo(HaveOccurred())
+			ip := r.nextip()
+			Expect(ip).NotTo(BeNil())
+
+			Expect(r.startIP).To(Equal(net.IP{192, 168, 2, 0}))
+		})
+	})
+
+	Context("when no lastReservedIP", func() {
+		It("should use the first IP of the first range as startIP after Next", func() {
+			a := newAllocatorWithMultiRanges()
+
+			// get range iterator and do the first Next
+			r, err := a.GetIter()
+			Expect(err).NotTo(HaveOccurred())
+			ip := r.nextip()
+			Expect(ip).NotTo(BeNil())
+
+			Expect(r.startIP).To(Equal(net.IP{192, 168, 1, 0}))
+		})
+	})
 })
 
 // nextip is a convenience function used for testing

plugins/ipam/host-local/backend/allocator/config.go

@@ -20,7 +20,9 @@ import (
 	"net"
 
 	"github.com/containernetworking/cni/pkg/types"
-	"github.com/containernetworking/cni/pkg/types/020"
+	"github.com/containernetworking/cni/pkg/version"
+
+	"github.com/containernetworking/plugins/pkg/ip"
 )
 
 // The top-level network config - IPAM plugins are passed the full configuration
@@ -29,8 +31,10 @@ type Net struct {
 	Name          string      `json:"name"`
 	CNIVersion    string      `json:"cniVersion"`
 	IPAM          *IPAMConfig `json:"ipam"`
-	RuntimeConfig struct {    // The capability arg
+	RuntimeConfig struct {
+		// The capability arg
 		IPRanges []RangeSet `json:"ipRanges,omitempty"`
+		IPs      []*ip.IP   `json:"ips,omitempty"`
 	} `json:"runtimeConfig,omitempty"`
 	Args *struct {
 		A *IPAMArgs `json:"cni"`
@@ -48,16 +52,16 @@ type IPAMConfig struct {
 	DataDir    string         `json:"dataDir"`
 	ResolvConf string         `json:"resolvConf"`
 	Ranges     []RangeSet     `json:"ranges"`
-	IPArgs     []net.IP       `json:"-"` // Requested IPs from CNI_ARGS and args
+	IPArgs     []net.IP       `json:"-"` // Requested IPs from CNI_ARGS, args and capabilities
 }
 
 type IPAMEnvArgs struct {
 	types.CommonArgs
-	IP net.IP `json:"ip,omitempty"`
+	IP ip.IP `json:"ip,omitempty"`
 }
 
 type IPAMArgs struct {
-	IPs []net.IP `json:"ips"`
+	IPs []*ip.IP `json:"ips"`
 }
 
 type RangeSet []Range
@@ -80,7 +84,7 @@ func LoadIPAMConfig(bytes []byte, envArgs string) (*IPAMConfig, string, error) {
 		return nil, "", fmt.Errorf("IPAM config missing 'ipam' key")
 	}
 
-	// Parse custom IP from both env args *and* the top-level args config
+	// parse custom IP from env args
 	if envArgs != "" {
 		e := IPAMEnvArgs{}
 		err := types.LoadArgs(envArgs, &e)
@@ -88,13 +92,23 @@ func LoadIPAMConfig(bytes []byte, envArgs string) (*IPAMConfig, string, error) {
 			return nil, "", err
 		}
 
-		if e.IP != nil {
-			n.IPAM.IPArgs = []net.IP{e.IP}
+		if e.IP.ToIP() != nil {
+			n.IPAM.IPArgs = []net.IP{e.IP.ToIP()}
 		}
 	}
 
+	// parse custom IPs from CNI args in network config
 	if n.Args != nil && n.Args.A != nil && len(n.Args.A.IPs) != 0 {
-		n.IPAM.IPArgs = append(n.IPAM.IPArgs, n.Args.A.IPs...)
+		for _, i := range n.Args.A.IPs {
+			n.IPAM.IPArgs = append(n.IPAM.IPArgs, i.ToIP())
+		}
+	}
+
+	// parse custom IPs from runtime configuration
+	if len(n.RuntimeConfig.IPs) > 0 {
+		for _, i := range n.RuntimeConfig.IPs {
+			n.IPAM.IPArgs = append(n.IPAM.IPArgs, i.ToIP())
+		}
 	}
 
 	for idx := range n.IPAM.IPArgs {
@@ -136,10 +150,8 @@ func LoadIPAMConfig(bytes []byte, envArgs string) (*IPAMConfig, string, error) {
 
 	// CNI spec 0.2.0 and below supported only one v4 and v6 address
 	if numV4 > 1 || numV6 > 1 {
-		for _, v := range types020.SupportedVersions {
-			if n.CNIVersion == v {
-				return nil, "", fmt.Errorf("CNI version %v does not support more than 1 address per family", n.CNIVersion)
-			}
+		if ok, _ := version.GreaterThanOrEqualTo(n.CNIVersion, "0.3.0"); !ok {
+			return nil, "", fmt.Errorf("CNI version %v does not support more than 1 address per family", n.CNIVersion)
 		}
 	}
 

plugins/ipam/host-local/backend/allocator/config_test.go

@@ -205,8 +205,9 @@ var _ = Describe("IPAM config", func() {
 		}))
 	})
 
-	It("Should parse CNI_ARGS env", func() {
-		input := `{
+	Context("Should parse CNI_ARGS env", func() {
+		It("without prefix", func() {
+			input := `{
 			"cniVersion": "0.3.1",
 			"name": "mynet",
 			"type": "ipvlan",
@@ -224,16 +225,43 @@ var _ = Describe("IPAM config", func() {
 			}
 		}`
 
-		envArgs := "IP=10.1.2.10"
+			envArgs := "IP=10.1.2.10"
 
-		conf, _, err := LoadIPAMConfig([]byte(input), envArgs)
-		Expect(err).NotTo(HaveOccurred())
-		Expect(conf.IPArgs).To(Equal([]net.IP{{10, 1, 2, 10}}))
+			conf, _, err := LoadIPAMConfig([]byte(input), envArgs)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(conf.IPArgs).To(Equal([]net.IP{{10, 1, 2, 10}}))
+		})
 
+		It("with prefix", func() {
+			input := `{
+			"cniVersion": "0.3.1",
+			"name": "mynet",
+			"type": "ipvlan",
+			"master": "foo0",
+			"ipam": {
+				"type": "host-local",
+				"ranges": [[
+					{
+						"subnet": "10.1.2.0/24",
+						"rangeStart": "10.1.2.9",
+						"rangeEnd": "10.1.2.20",
+						"gateway": "10.1.2.30"
+					}
+				]]
+			}
+		}`
+
+			envArgs := "IP=10.1.2.11/24"
+
+			conf, _, err := LoadIPAMConfig([]byte(input), envArgs)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(conf.IPArgs).To(Equal([]net.IP{{10, 1, 2, 11}}))
+		})
 	})
 
-	It("Should parse config args", func() {
-		input := `{
+	Context("Should parse config args", func() {
+		It("without prefix", func() {
+			input := `{
 			"cniVersion": "0.3.1",
 			"name": "mynet",
 			"type": "ipvlan",
@@ -265,16 +293,62 @@ var _ = Describe("IPAM config", func() {
 			}
 		}`
 
-		envArgs := "IP=10.1.2.10"
+			envArgs := "IP=10.1.2.10"
 
-		conf, _, err := LoadIPAMConfig([]byte(input), envArgs)
-		Expect(err).NotTo(HaveOccurred())
-		Expect(conf.IPArgs).To(Equal([]net.IP{
-			{10, 1, 2, 10},
-			{10, 1, 2, 11},
-			{11, 11, 11, 11},
-			net.ParseIP("2001:db8:1::11"),
-		}))
+			conf, _, err := LoadIPAMConfig([]byte(input), envArgs)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(conf.IPArgs).To(Equal([]net.IP{
+				{10, 1, 2, 10},
+				{10, 1, 2, 11},
+				{11, 11, 11, 11},
+				net.ParseIP("2001:db8:1::11"),
+			}))
+		})
+
+		It("with prefix", func() {
+			input := `{
+			"cniVersion": "0.3.1",
+			"name": "mynet",
+			"type": "ipvlan",
+			"master": "foo0",
+			"args": {
+				"cni": {
+					"ips": [ "10.1.2.11/24", "11.11.11.11/24", "2001:db8:1::11/64"]
+				}
+			},
+			"ipam": {
+				"type": "host-local",
+				"ranges": [
+					[{
+						"subnet": "10.1.2.0/24",
+						"rangeStart": "10.1.2.9",
+						"rangeEnd": "10.1.2.20",
+						"gateway": "10.1.2.30"
+					}],
+					[{
+						"subnet": "11.1.2.0/24",
+						"rangeStart": "11.1.2.9",
+						"rangeEnd": "11.1.2.20",
+						"gateway": "11.1.2.30"
+					}],
+					[{
+						"subnet": "2001:db8:1::/64"
+					}]
+				]
+			}
+		}`
+
+			envArgs := "IP=10.1.2.10/24"
+
+			conf, _, err := LoadIPAMConfig([]byte(input), envArgs)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(conf.IPArgs).To(Equal([]net.IP{
+				{10, 1, 2, 10},
+				{10, 1, 2, 11},
+				{11, 11, 11, 11},
+				net.ParseIP("2001:db8:1::11"),
+			}))
+		})
 	})
 
 	It("Should detect overlap between rangesets", func() {
@@ -379,4 +453,29 @@ var _ = Describe("IPAM config", func() {
 		_, _, err := LoadIPAMConfig([]byte(input), "")
 		Expect(err).NotTo(HaveOccurred())
 	})
+
+	It("Should parse custom IPs from runtime configuration", func() {
+		input := `{
+			"cniVersion": "0.3.1",
+			"name": "mynet",
+			"type": "ipvlan",
+			"master": "foo0",
+			"runtimeConfig": {
+				"ips": ["192.168.0.1", "192.168.0.5/24", "2001:db8::1/64"]
+			},
+			"ipam": {
+				"type": "host-local",
+				"subnet": "10.1.2.0/24"
+			}
+		}`
+		conf, version, err := LoadIPAMConfig([]byte(input), "")
+		Expect(err).NotTo(HaveOccurred())
+		Expect(version).Should(Equal("0.3.1"))
+
+		Expect(conf.IPArgs).To(Equal([]net.IP{
+			net.IPv4(192, 168, 0, 1).To4(),
+			net.IPv4(192, 168, 0, 5).To4(),
+			net.ParseIP("2001:db8::1"),
+		}))
+	})
 })

plugins/ipam/host-local/backend/disk/backend.go

@@ -15,7 +15,6 @@
 package disk
 
 import (
-	"io/ioutil"
 	"net"
 	"os"
 	"path/filepath"
@@ -77,7 +76,7 @@ func (s *Store) Reserve(id string, ifname string, ip net.IP, rangeID string) (bo
 	}
 	// store the reserved ip in lastIPFile
 	ipfile := GetEscapedPath(s.dataDir, lastIPFilePrefix+rangeID)
-	err = ioutil.WriteFile(ipfile, []byte(ip.String()), 0644)
+	err = os.WriteFile(ipfile, []byte(ip.String()), 0644)
 	if err != nil {
 		return false, err
 	}
@@ -87,17 +86,13 @@ func (s *Store) Reserve(id string, ifname string, ip net.IP, rangeID string) (bo
 // LastReservedIP returns the last reserved IP if exists
 func (s *Store) LastReservedIP(rangeID string) (net.IP, error) {
 	ipfile := GetEscapedPath(s.dataDir, lastIPFilePrefix+rangeID)
-	data, err := ioutil.ReadFile(ipfile)
+	data, err := os.ReadFile(ipfile)
 	if err != nil {
 		return nil, err
 	}
 	return net.ParseIP(string(data)), nil
 }
 
-func (s *Store) Release(ip net.IP) error {
-	return os.Remove(GetEscapedPath(s.dataDir, ip.String()))
-}
-
 func (s *Store) FindByKey(id string, ifname string, match string) (bool, error) {
 	found := false
 
@@ -105,7 +100,7 @@ func (s *Store) FindByKey(id string, ifname string, match string) (bool, error)
 		if err != nil || info.IsDir() {
 			return nil
 		}
-		data, err := ioutil.ReadFile(path)
+		data, err := os.ReadFile(path)
 		if err != nil {
 			return nil
 		}
@@ -141,7 +136,7 @@ func (s *Store) ReleaseByKey(id string, ifname string, match string) (bool, erro
 		if err != nil || info.IsDir() {
 			return nil
 		}
-		data, err := ioutil.ReadFile(path)
+		data, err := os.ReadFile(path)
 		if err != nil {
 			return nil
 		}
@@ -185,7 +180,7 @@ func (s *Store) GetByID(id string, ifname string) []net.IP {
 		if err != nil || info.IsDir() {
 			return nil
 		}
-		data, err := ioutil.ReadFile(path)
+		data, err := os.ReadFile(path)
 		if err != nil {
 			return nil
 		}

plugins/ipam/host-local/backend/disk/lock_test.go

@@ -15,7 +15,6 @@
 package disk
 
 import (
-	"io/ioutil"
 	"os"
 	"path/filepath"
 
@@ -25,7 +24,7 @@ import (
 
 var _ = Describe("Lock Operations", func() {
 	It("locks a file path", func() {
-		dir, err := ioutil.TempDir("", "")
+		dir, err := os.MkdirTemp("", "")
 		Expect(err).ToNot(HaveOccurred())
 		defer os.RemoveAll(dir)
 
@@ -47,7 +46,7 @@ var _ = Describe("Lock Operations", func() {
 	})
 
 	It("locks a folder path", func() {
-		dir, err := ioutil.TempDir("", "")
+		dir, err := os.MkdirTemp("", "")
 		Expect(err).ToNot(HaveOccurred())
 		defer os.RemoveAll(dir)
 

plugins/ipam/host-local/backend/store.go

@@ -22,7 +22,6 @@ type Store interface {
 	Close() error
 	Reserve(id string, ifname string, ip net.IP, rangeID string) (bool, error)
 	LastReservedIP(rangeID string) (net.IP, error)
-	Release(ip net.IP) error
 	ReleaseByID(id string, ifname string) error
 	GetByID(id string, ifname string) []net.IP
 }

plugins/ipam/host-local/backend/testing/fake_store.go

@@ -63,11 +63,6 @@ func (s *FakeStore) LastReservedIP(rangeID string) (net.IP, error) {
 	return ip, nil
 }
 
-func (s *FakeStore) Release(ip net.IP) error {
-	delete(s.ipMap, ip.String())
-	return nil
-}
-
 func (s *FakeStore) ReleaseByID(id string, ifname string) error {
 	toDelete := []string{}
 	for k, v := range s.ipMap {

plugins/ipam/host-local/dns_test.go

@@ -15,7 +15,6 @@
 package main
 
 import (
-	"io/ioutil"
 	"os"
 
 	"github.com/containernetworking/cni/pkg/types"
@@ -64,7 +63,7 @@ options four
 })
 
 func parse(contents string) (*types.DNS, error) {
-	f, err := ioutil.TempFile("", "host_local_resolv")
+	f, err := os.CreateTemp("", "host_local_resolv")
 	if err != nil {
 		return nil, err
 	}

plugins/ipam/host-local/main.go

@@ -25,7 +25,7 @@ import (
 
 	"github.com/containernetworking/cni/pkg/skel"
 	"github.com/containernetworking/cni/pkg/types"
-	"github.com/containernetworking/cni/pkg/types/current"
+	current "github.com/containernetworking/cni/pkg/types/100"
 	"github.com/containernetworking/cni/pkg/version"
 )
 
@@ -62,7 +62,7 @@ func cmdAdd(args *skel.CmdArgs) error {
 		return err
 	}
 
-	result := &current.Result{}
+	result := &current.Result{CNIVersion: current.ImplementedSpecVersion}
 
 	if ipamConf.ResolvConf != "" {
 		dns, err := parseResolvConf(ipamConf.ResolvConf)

plugins/ipam/static/README.md

@@ -1,4 +1,4 @@
 
 This document has moved to the [containernetworking/cni.dev](https://github.com/containernetworking/cni.dev) repo.
 
-You can find it online here: https://cni.dev/plugins/ipam/static/
+You can find it online here: https://cni.dev/plugins/current/ipam/static/

plugins/ipam/static/main.go

@@ -22,8 +22,7 @@ import (
 
 	"github.com/containernetworking/cni/pkg/skel"
 	"github.com/containernetworking/cni/pkg/types"
-	types020 "github.com/containernetworking/cni/pkg/types/020"
-	"github.com/containernetworking/cni/pkg/types/current"
+	current "github.com/containernetworking/cni/pkg/types/100"
 	"github.com/containernetworking/cni/pkg/version"
 	bv "github.com/containernetworking/plugins/pkg/utils/buildversion"
 )
@@ -144,6 +143,9 @@ func LoadIPAMConfig(bytes []byte, envArgs string) (*IPAMConfig, string, error) {
 	if err := json.Unmarshal(bytes, &n); err != nil {
 		return nil, "", err
 	}
+	if n.IPAM == nil {
+		return nil, "", fmt.Errorf("IPAM config missing 'ipam' key")
+	}
 
 	// load IP from CNI_ARGS
 	if envArgs != "" {
@@ -159,7 +161,7 @@ func LoadIPAMConfig(bytes []byte, envArgs string) (*IPAMConfig, string, error) {
 
 				ip, subnet, err := net.ParseCIDR(ipstr)
 				if err != nil {
-					return nil, "", fmt.Errorf("invalid CIDR %s: %s", ipstr, err)
+					return nil, "", fmt.Errorf("the 'ip' field is expected to be in CIDR notation, got: '%s'", ipstr)
 				}
 
 				addr := Address{
@@ -190,8 +192,13 @@ func LoadIPAMConfig(bytes []byte, envArgs string) (*IPAMConfig, string, error) {
 	if n.Args != nil && n.Args.A != nil && len(n.Args.A.IPs) != 0 {
 		// args IP overwrites IP, so clear IPAM Config
 		n.IPAM.Addresses = make([]Address, 0, len(n.Args.A.IPs))
-		for _, addr := range n.Args.A.IPs {
-			n.IPAM.Addresses = append(n.IPAM.Addresses, Address{AddressStr: addr})
+		for _, addrStr := range n.Args.A.IPs {
+			ip, addr, err := net.ParseCIDR(addrStr)
+			if err != nil {
+				return nil, "", fmt.Errorf("an entry in the 'ips' field is NOT in CIDR notation, got: '%s'", addrStr)
+			}
+			addr.IP = ip
+			n.IPAM.Addresses = append(n.IPAM.Addresses, Address{AddressStr: addrStr, Address: *addr})
 		}
 	}
 
@@ -199,46 +206,46 @@ func LoadIPAMConfig(bytes []byte, envArgs string) (*IPAMConfig, string, error) {
 	if len(n.RuntimeConfig.IPs) != 0 {
 		// runtimeConfig IP overwrites IP, so clear IPAM Config
 		n.IPAM.Addresses = make([]Address, 0, len(n.RuntimeConfig.IPs))
-		for _, addr := range n.RuntimeConfig.IPs {
-			n.IPAM.Addresses = append(n.IPAM.Addresses, Address{AddressStr: addr})
+		for _, addrStr := range n.RuntimeConfig.IPs {
+			ip, addr, err := net.ParseCIDR(addrStr)
+			if err != nil {
+				return nil, "", fmt.Errorf("an entry in the 'ips' field is NOT in CIDR notation, got: '%s'", addrStr)
+			}
+			addr.IP = ip
+			n.IPAM.Addresses = append(n.IPAM.Addresses, Address{AddressStr: addrStr, Address: *addr})
 		}
 	}
 
-	if n.IPAM == nil {
-		return nil, "", fmt.Errorf("IPAM config missing 'ipam' key")
-	}
-
 	// Validate all ranges
 	numV4 := 0
 	numV6 := 0
 
 	for i := range n.IPAM.Addresses {
-		ip, addr, err := net.ParseCIDR(n.IPAM.Addresses[i].AddressStr)
-		if err != nil {
-			return nil, "", fmt.Errorf("invalid CIDR %s: %s", n.IPAM.Addresses[i].AddressStr, err)
+		if n.IPAM.Addresses[i].Address.IP == nil {
+			ip, addr, err := net.ParseCIDR(n.IPAM.Addresses[i].AddressStr)
+			if err != nil {
+				return nil, "", fmt.Errorf(
+					"the 'address' field is expected to be in CIDR notation, got: '%s'", n.IPAM.Addresses[i].AddressStr)
+			}
+			n.IPAM.Addresses[i].Address = *addr
+			n.IPAM.Addresses[i].Address.IP = ip
 		}
-		n.IPAM.Addresses[i].Address = *addr
-		n.IPAM.Addresses[i].Address.IP = ip
 
 		if err := canonicalizeIP(&n.IPAM.Addresses[i].Address.IP); err != nil {
 			return nil, "", fmt.Errorf("invalid address %d: %s", i, err)
 		}
 
 		if n.IPAM.Addresses[i].Address.IP.To4() != nil {
-			n.IPAM.Addresses[i].Version = "4"
 			numV4++
 		} else {
-			n.IPAM.Addresses[i].Version = "6"
 			numV6++
 		}
 	}
 
 	// CNI spec 0.2.0 and below supported only one v4 and v6 address
 	if numV4 > 1 || numV6 > 1 {
-		for _, v := range types020.SupportedVersions {
-			if n.CNIVersion == v {
-				return nil, "", fmt.Errorf("CNI version %v does not support more than 1 address per family", n.CNIVersion)
-			}
+		if ok, _ := version.GreaterThanOrEqualTo(n.CNIVersion, "0.3.0"); !ok {
+			return nil, "", fmt.Errorf("CNI version %v does not support more than 1 address per family", n.CNIVersion)
 		}
 	}
 
@@ -254,14 +261,16 @@ func cmdAdd(args *skel.CmdArgs) error {
 		return err
 	}
 
-	result := &current.Result{}
-	result.DNS = ipamConf.DNS
-	result.Routes = ipamConf.Routes
+	result := &current.Result{
+		CNIVersion: current.ImplementedSpecVersion,
+		DNS:        ipamConf.DNS,
+		Routes:     ipamConf.Routes,
+	}
 	for _, v := range ipamConf.Addresses {
 		result.IPs = append(result.IPs, &current.IPConfig{
-			Version: v.Version,
 			Address: v.Address,
-			Gateway: v.Gateway})
+			Gateway: v.Gateway,
+		})
 	}
 
 	return types.PrintResult(result, confVersion)

plugins/linux_only.txt

@@ -6,6 +6,7 @@ plugins/main/loopback
 plugins/main/macvlan
 plugins/main/ptp
 plugins/main/vlan
+plugins/main/dummy
 plugins/meta/portmap
 plugins/meta/tuning
 plugins/meta/bandwidth

plugins/main/bridge/README.md

@@ -1,5 +1,5 @@
 
 This document has moved to the [containernetworking/cni.dev](https://github.com/containernetworking/cni.dev) repo.
 
-You can find it online here: https://cni.dev/plugins/main/bridge/
+You can find it online here: https://cni.dev/plugins/current/main/bridge/
 

plugins/main/bridge/bridge.go

@@ -18,21 +18,21 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"io/ioutil"
 	"net"
+	"os"
 	"runtime"
 	"syscall"
 	"time"
 
-	"github.com/j-keck/arping"
 	"github.com/vishvananda/netlink"
 
 	"github.com/containernetworking/cni/pkg/skel"
 	"github.com/containernetworking/cni/pkg/types"
-	"github.com/containernetworking/cni/pkg/types/current"
+	current "github.com/containernetworking/cni/pkg/types/100"
 	"github.com/containernetworking/cni/pkg/version"
 	"github.com/containernetworking/plugins/pkg/ip"
 	"github.com/containernetworking/plugins/pkg/ipam"
+	"github.com/containernetworking/plugins/pkg/link"
 	"github.com/containernetworking/plugins/pkg/ns"
 	"github.com/containernetworking/plugins/pkg/utils"
 	bv "github.com/containernetworking/plugins/pkg/utils/buildversion"
@@ -55,6 +55,27 @@ type NetConf struct {
 	HairpinMode  bool   `json:"hairpinMode"`
 	PromiscMode  bool   `json:"promiscMode"`
 	Vlan         int    `json:"vlan"`
+	MacSpoofChk  bool   `json:"macspoofchk,omitempty"`
+	EnableDad    bool   `json:"enabledad,omitempty"`
+
+	Args struct {
+		Cni BridgeArgs `json:"cni,omitempty"`
+	} `json:"args,omitempty"`
+	RuntimeConfig struct {
+		Mac string `json:"mac,omitempty"`
+	} `json:"runtimeConfig,omitempty"`
+
+	mac string
+}
+
+type BridgeArgs struct {
+	Mac string `json:"mac,omitempty"`
+}
+
+// MacEnvArgs represents CNI_ARGS
+type MacEnvArgs struct {
+	types.CommonArgs
+	MAC types.UnmarshallableString `json:"mac,omitempty"`
 }
 
 type gwInfo struct {
@@ -70,7 +91,7 @@ func init() {
 	runtime.LockOSThread()
 }
 
-func loadNetConf(bytes []byte) (*NetConf, string, error) {
+func loadNetConf(bytes []byte, envArgs string) (*NetConf, string, error) {
 	n := &NetConf{
 		BrName: defaultBrName,
 	}
@@ -80,13 +101,33 @@ func loadNetConf(bytes []byte) (*NetConf, string, error) {
 	if n.Vlan < 0 || n.Vlan > 4094 {
 		return nil, "", fmt.Errorf("invalid VLAN ID %d (must be between 0 and 4094)", n.Vlan)
 	}
+
+	if envArgs != "" {
+		e := MacEnvArgs{}
+		if err := types.LoadArgs(envArgs, &e); err != nil {
+			return nil, "", err
+		}
+
+		if e.MAC != "" {
+			n.mac = string(e.MAC)
+		}
+	}
+
+	if mac := n.Args.Cni.Mac; mac != "" {
+		n.mac = mac
+	}
+
+	if mac := n.RuntimeConfig.Mac; mac != "" {
+		n.mac = mac
+	}
+
 	return n, n.CNIVersion, nil
 }
 
 // calcGateways processes the results from the IPAM plugin and does the
 // following for each IP family:
-//    - Calculates and compiles a list of gateway addresses
-//    - Adds a default route if needed
+//   - Calculates and compiles a list of gateway addresses
+//   - Adds a default route if needed
 func calcGateways(result *current.Result, n *NetConf) (*gwInfo, *gwInfo, error) {
 
 	gwsV4 := &gwInfo{}
@@ -273,7 +314,7 @@ func ensureVlanInterface(br *netlink.Bridge, vlanId int) (netlink.Link, error) {
 			return nil, fmt.Errorf("faild to find host namespace: %v", err)
 		}
 
-		_, brGatewayIface, err := setupVeth(hostNS, br, name, br.MTU, false, vlanId)
+		_, brGatewayIface, err := setupVeth(hostNS, br, name, br.MTU, false, vlanId, "")
 		if err != nil {
 			return nil, fmt.Errorf("faild to create vlan gateway %q: %v", name, err)
 		}
@@ -282,18 +323,23 @@ func ensureVlanInterface(br *netlink.Bridge, vlanId int) (netlink.Link, error) {
 		if err != nil {
 			return nil, fmt.Errorf("failed to lookup %q: %v", brGatewayIface.Name, err)
 		}
+
+		err = netlink.LinkSetUp(brGatewayVeth)
+		if err != nil {
+			return nil, fmt.Errorf("failed to up %q: %v", brGatewayIface.Name, err)
+		}
 	}
 
 	return brGatewayVeth, nil
 }
 
-func setupVeth(netns ns.NetNS, br *netlink.Bridge, ifName string, mtu int, hairpinMode bool, vlanID int) (*current.Interface, *current.Interface, error) {
+func setupVeth(netns ns.NetNS, br *netlink.Bridge, ifName string, mtu int, hairpinMode bool, vlanID int, mac string) (*current.Interface, *current.Interface, error) {
 	contIface := &current.Interface{}
 	hostIface := &current.Interface{}
 
 	err := netns.Do(func(hostNS ns.NetNS) error {
 		// create the veth pair in the container and move host end into host netns
-		hostVeth, containerVeth, err := ip.SetupVeth(ifName, mtu, hostNS)
+		hostVeth, containerVeth, err := ip.SetupVeth(ifName, mtu, mac, hostNS)
 		if err != nil {
 			return err
 		}
@@ -340,10 +386,7 @@ func calcGatewayIP(ipn *net.IPNet) net.IP {
 }
 
 func setupBridge(n *NetConf) (*netlink.Bridge, *current.Interface, error) {
-	vlanFiltering := false
-	if n.Vlan != 0 {
-		vlanFiltering = true
-	}
+	vlanFiltering := n.Vlan != 0
 	// create bridge if necessary
 	br, err := ensureBridge(n.BrName, n.MTU, n.PromiscMode, vlanFiltering)
 	if err != nil {
@@ -356,20 +399,6 @@ func setupBridge(n *NetConf) (*netlink.Bridge, *current.Interface, error) {
 	}, nil
 }
 
-// disableIPV6DAD disables IPv6 Duplicate Address Detection (DAD)
-// for an interface, if the interface does not support enhanced_dad.
-// We do this because interfaces with hairpin mode will see their own DAD packets
-func disableIPV6DAD(ifName string) error {
-	// ehanced_dad sends a nonce with the DAD packets, so that we can safely
-	// ignore ourselves
-	enh, err := ioutil.ReadFile(fmt.Sprintf("/proc/sys/net/ipv6/conf/%s/enhanced_dad", ifName))
-	if err == nil && string(enh) == "1\n" {
-		return nil
-	}
-	f := fmt.Sprintf("/proc/sys/net/ipv6/conf/%s/accept_dad", ifName)
-	return ioutil.WriteFile(f, []byte("0"), 0644)
-}
-
 func enableIPForward(family int) error {
 	if family == netlink.FAMILY_V4 {
 		return ip.EnableIP4Forward()
@@ -380,7 +409,7 @@ func enableIPForward(family int) error {
 func cmdAdd(args *skel.CmdArgs) error {
 	var success bool = false
 
-	n, cniVersion, err := loadNetConf(args.StdinData)
+	n, cniVersion, err := loadNetConf(args.StdinData, args.Args)
 	if err != nil {
 		return err
 	}
@@ -392,7 +421,7 @@ func cmdAdd(args *skel.CmdArgs) error {
 	}
 
 	if n.HairpinMode && n.PromiscMode {
-		return fmt.Errorf("cannot set hairpin mode and promiscous mode at the same time.")
+		return fmt.Errorf("cannot set hairpin mode and promiscuous mode at the same time.")
 	}
 
 	br, brInterface, err := setupBridge(n)
@@ -406,13 +435,34 @@ func cmdAdd(args *skel.CmdArgs) error {
 	}
 	defer netns.Close()
 
-	hostInterface, containerInterface, err := setupVeth(netns, br, args.IfName, n.MTU, n.HairpinMode, n.Vlan)
+	hostInterface, containerInterface, err := setupVeth(netns, br, args.IfName, n.MTU, n.HairpinMode, n.Vlan, n.mac)
 	if err != nil {
 		return err
 	}
 
 	// Assume L2 interface only
-	result := &current.Result{CNIVersion: cniVersion, Interfaces: []*current.Interface{brInterface, hostInterface, containerInterface}}
+	result := &current.Result{
+		CNIVersion: current.ImplementedSpecVersion,
+		Interfaces: []*current.Interface{
+			brInterface,
+			hostInterface,
+			containerInterface,
+		},
+	}
+
+	if n.MacSpoofChk {
+		sc := link.NewSpoofChecker(hostInterface.Name, containerInterface.Mac, uniqueID(args.ContainerID, args.IfName))
+		if err := sc.Setup(); err != nil {
+			return err
+		}
+		defer func() {
+			if !success {
+				if err := sc.Teardown(); err != nil {
+					fmt.Fprintf(os.Stderr, "%v", err)
+				}
+			}
+		}()
+	}
 
 	if isLayer3 {
 		// run the IPAM plugin and get back the config to apply
@@ -436,6 +486,7 @@ func cmdAdd(args *skel.CmdArgs) error {
 
 		result.IPs = ipamResult.IPs
 		result.Routes = ipamResult.Routes
+		result.DNS = ipamResult.DNS
 
 		if len(result.IPs) == 0 {
 			return errors.New("IPAM plugin returned missing IP config")
@@ -449,17 +500,13 @@ func cmdAdd(args *skel.CmdArgs) error {
 
 		// Configure the container hardware address and IP address(es)
 		if err := netns.Do(func(_ ns.NetNS) error {
-			// Disable IPv6 DAD just in case hairpin mode is enabled on the
-			// bridge. Hairpin mode causes echos of neighbor solicitation
-			// packets, which causes DAD failures.
-			for _, ipc := range result.IPs {
-				if ipc.Version == "6" && (n.HairpinMode || n.PromiscMode) {
-					if err := disableIPV6DAD(args.IfName); err != nil {
-						return err
-					}
-					break
-				}
+			if n.EnableDad {
+				_, _ = sysctl.Sysctl(fmt.Sprintf("/net/ipv6/conf/%s/enhanced_dad", args.IfName), "1")
+				_, _ = sysctl.Sysctl(fmt.Sprintf("net/ipv6/conf/%s/accept_dad", args.IfName), "1")
+			} else {
+				_, _ = sysctl.Sysctl(fmt.Sprintf("net/ipv6/conf/%s/accept_dad", args.IfName), "0")
 			}
+			_, _ = sysctl.Sysctl(fmt.Sprintf("net/ipv4/conf/%s/arp_notify", args.IfName), "1")
 
 			// Add the IP to the interface
 			if err := ipam.ConfigureIface(args.IfName, result); err != nil {
@@ -470,41 +517,6 @@ func cmdAdd(args *skel.CmdArgs) error {
 			return err
 		}
 
-		// check bridge port state
-		retries := []int{0, 50, 500, 1000, 1000}
-		for idx, sleep := range retries {
-			time.Sleep(time.Duration(sleep) * time.Millisecond)
-
-			hostVeth, err := netlink.LinkByName(hostInterface.Name)
-			if err != nil {
-				return err
-			}
-			if hostVeth.Attrs().OperState == netlink.OperUp {
-				break
-			}
-
-			if idx == len(retries)-1 {
-				return fmt.Errorf("bridge port in error state: %s", hostVeth.Attrs().OperState)
-			}
-		}
-
-		// Send a gratuitous arp
-		if err := netns.Do(func(_ ns.NetNS) error {
-			contVeth, err := net.InterfaceByName(args.IfName)
-			if err != nil {
-				return err
-			}
-
-			for _, ipc := range result.IPs {
-				if ipc.Version == "4" {
-					_ = arping.GratuitousArpOverIface(ipc.Address.IP, *contVeth)
-				}
-			}
-			return nil
-		}); err != nil {
-			return err
-		}
-
 		if n.IsGW {
 			var firstV4Addr net.IP
 			var vlanInterface *current.Interface
@@ -555,8 +567,45 @@ func cmdAdd(args *skel.CmdArgs) error {
 				}
 			}
 		}
+	} else {
+		if err := netns.Do(func(_ ns.NetNS) error {
+			link, err := netlink.LinkByName(args.IfName)
+			if err != nil {
+				return fmt.Errorf("failed to retrieve link: %v", err)
+			}
+			// If layer 2 we still need to set the container veth to up
+			if err = netlink.LinkSetUp(link); err != nil {
+				return fmt.Errorf("failed to set %q up: %v", args.IfName, err)
+			}
+			return nil
+		}); err != nil {
+			return err
+		}
 	}
 
+	var hostVeth netlink.Link
+
+	// check bridge port state
+	retries := []int{0, 50, 500, 1000, 1000}
+	for idx, sleep := range retries {
+		time.Sleep(time.Duration(sleep) * time.Millisecond)
+
+		hostVeth, err = netlink.LinkByName(hostInterface.Name)
+		if err != nil {
+			return err
+		}
+		if hostVeth.Attrs().OperState == netlink.OperUp {
+			break
+		}
+
+		if idx == len(retries)-1 {
+			return fmt.Errorf("bridge port in error state: %s", hostVeth.Attrs().OperState)
+		}
+	}
+
+	// In certain circumstances, the host-side of the veth may change addrs
+	hostInterface.Mac = hostVeth.Attrs().HardwareAddr.String()
+
 	// Refetch the bridge since its MAC address may change when the first
 	// veth is added or after its IP address is set
 	br, err = bridgeByName(n.BrName)
@@ -565,34 +614,48 @@ func cmdAdd(args *skel.CmdArgs) error {
 	}
 	brInterface.Mac = br.Attrs().HardwareAddr.String()
 
-	result.DNS = n.DNS
-
 	// Return an error requested by testcases, if any
 	if debugPostIPAMError != nil {
 		return debugPostIPAMError
 	}
 
+	// Use incoming DNS settings if provided, otherwise use the
+	// settings that were already configued by the IPAM plugin
+	if dnsConfSet(n.DNS) {
+		result.DNS = n.DNS
+	}
+
 	success = true
 
 	return types.PrintResult(result, cniVersion)
 }
 
+func dnsConfSet(dnsConf types.DNS) bool {
+	return dnsConf.Nameservers != nil ||
+		dnsConf.Search != nil ||
+		dnsConf.Options != nil ||
+		dnsConf.Domain != ""
+}
+
 func cmdDel(args *skel.CmdArgs) error {
-	n, _, err := loadNetConf(args.StdinData)
+	n, _, err := loadNetConf(args.StdinData, args.Args)
 	if err != nil {
 		return err
 	}
 
 	isLayer3 := n.IPAM.Type != ""
 
-	if isLayer3 {
-		if err := ipam.ExecDel(n.IPAM.Type, args.StdinData); err != nil {
-			return err
+	ipamDel := func() error {
+		if isLayer3 {
+			if err := ipam.ExecDel(n.IPAM.Type, args.StdinData); err != nil {
+				return err
+			}
 		}
+		return nil
 	}
 
 	if args.Netns == "" {
-		return nil
+		return ipamDel()
 	}
 
 	// There is a netns so try to clean up. Delete can be called multiple times
@@ -609,9 +672,28 @@ func cmdDel(args *skel.CmdArgs) error {
 	})
 
 	if err != nil {
+		//  if NetNs is passed down by the Cloud Orchestration Engine, or if it called multiple times
+		// so don't return an error if the device is already removed.
+		// https://github.com/kubernetes/kubernetes/issues/43014#issuecomment-287164444
+		_, ok := err.(ns.NSPathNotExistErr)
+		if ok {
+			return ipamDel()
+		}
 		return err
 	}
 
+	// call ipam.ExecDel after clean up device in netns
+	if err := ipamDel(); err != nil {
+		return err
+	}
+
+	if n.MacSpoofChk {
+		sc := link.NewSpoofChecker("", "", uniqueID(args.ContainerID, args.IfName))
+		if err := sc.Teardown(); err != nil {
+			fmt.Fprintf(os.Stderr, "%v", err)
+		}
+	}
+
 	if isLayer3 && n.IPMasq {
 		chain := utils.FormatChainName(n.Name, args.ContainerID)
 		comment := utils.FormatComment(n.Name, args.ContainerID)
@@ -769,7 +851,7 @@ func validateCniContainerInterface(intf current.Interface) (cniBridgeIf, error)
 
 func cmdCheck(args *skel.CmdArgs) error {
 
-	n, _, err := loadNetConf(args.StdinData)
+	n, _, err := loadNetConf(args.StdinData, args.Args)
 	if err != nil {
 		return err
 	}
@@ -892,3 +974,7 @@ func cmdCheck(args *skel.CmdArgs) error {
 
 	return nil
 }
+
+func uniqueID(containerID, cniIface string) string {
+	return containerID + "-" + cniIface
+}

plugins/main/dummy/README.md

@@ -0,0 +1,39 @@
+---
+title: dummy plugin
+description: "plugins/main/dummy/README.md"
+date: 2022-05-12
+toc: true
+draft: true
+weight: 200
+---
+
+## Overview
+
+dummy is a useful feature for routing packets through the Linux kernel without transmitting.
+
+Like loopback, it is a purely virtual interface that allows packets to be routed to a designated IP address. Unlike loopback, the IP address can be arbitrary and is not restricted to the `127.0.0.0/8` range.
+
+## Example configuration
+
+```json
+{
+	"name": "mynet",
+	"type": "dummy",
+	"ipam": {
+		"type": "host-local",
+		"subnet": "10.1.2.0/24"
+	}
+}
+```
+
+## Network configuration reference
+
+* `name` (string, required): the name of the network.
+* `type` (string, required): "dummy".
+* `ipam` (dictionary, required): IPAM configuration to be used for this network.
+
+## Notes
+
+* `dummy` does not transmit packets.
+Therefore the container will not be able to reach any external network.
+This solution is designed to be used in conjunction with other CNI plugins (e.g., `bridge`) to provide an internal non-loopback address for applications to use.

plugins/main/dummy/dummy.go

@@ -0,0 +1,300 @@
+// Copyright 2022 Arista Networks
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net"
+
+	"github.com/vishvananda/netlink"
+
+	"github.com/containernetworking/cni/pkg/skel"
+	"github.com/containernetworking/cni/pkg/types"
+	current "github.com/containernetworking/cni/pkg/types/100"
+	"github.com/containernetworking/cni/pkg/version"
+
+	"github.com/containernetworking/plugins/pkg/ip"
+	"github.com/containernetworking/plugins/pkg/ipam"
+	"github.com/containernetworking/plugins/pkg/ns"
+	bv "github.com/containernetworking/plugins/pkg/utils/buildversion"
+)
+
+func parseNetConf(bytes []byte) (*types.NetConf, error) {
+	conf := &types.NetConf{}
+	if err := json.Unmarshal(bytes, conf); err != nil {
+		return nil, fmt.Errorf("failed to parse network config: %v", err)
+	}
+	return conf, nil
+}
+
+func createDummy(conf *types.NetConf, ifName string, netns ns.NetNS) (*current.Interface, error) {
+
+	dummy := &current.Interface{}
+
+	dm := &netlink.Dummy{
+		LinkAttrs: netlink.LinkAttrs{
+			Name:      ifName,
+			Namespace: netlink.NsFd(int(netns.Fd())),
+		},
+	}
+
+	if err := netlink.LinkAdd(dm); err != nil {
+		return nil, fmt.Errorf("failed to create dummy: %v", err)
+	}
+	dummy.Name = ifName
+
+	err := netns.Do(func(_ ns.NetNS) error {
+		// Re-fetch interface to get all properties/attributes
+		contDummy, err := netlink.LinkByName(ifName)
+		if err != nil {
+			return fmt.Errorf("failed to fetch dummy%q: %v", ifName, err)
+		}
+
+		dummy.Mac = contDummy.Attrs().HardwareAddr.String()
+		dummy.Sandbox = netns.Path()
+
+		return nil
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	return dummy, nil
+}
+
+func cmdAdd(args *skel.CmdArgs) error {
+	conf, err := parseNetConf(args.StdinData)
+	if err != nil {
+		return err
+	}
+
+	if conf.IPAM.Type == "" {
+		return errors.New("dummy interface requires an IPAM configuration")
+	}
+
+	netns, err := ns.GetNS(args.Netns)
+	if err != nil {
+		return fmt.Errorf("failed to open netns %q: %v", netns, err)
+	}
+	defer netns.Close()
+
+	dummyInterface, err := createDummy(conf, args.IfName, netns)
+	if err != nil {
+		return err
+	}
+
+	// Delete link if err to avoid link leak in this ns
+	defer func() {
+		if err != nil {
+			netns.Do(func(_ ns.NetNS) error {
+				return ip.DelLinkByName(args.IfName)
+			})
+		}
+	}()
+
+	r, err := ipam.ExecAdd(conf.IPAM.Type, args.StdinData)
+	if err != nil {
+		return err
+	}
+
+	// defer ipam deletion to avoid ip leak
+	defer func() {
+		if err != nil {
+			ipam.ExecDel(conf.IPAM.Type, args.StdinData)
+		}
+	}()
+
+	// convert IPAMResult to current Result type
+	result, err := current.NewResultFromResult(r)
+	if err != nil {
+		return err
+	}
+
+	if len(result.IPs) == 0 {
+		return errors.New("IPAM plugin returned missing IP config")
+	}
+
+	for _, ipc := range result.IPs {
+		// all addresses apply to the container dummy interface
+		ipc.Interface = current.Int(0)
+	}
+
+	result.Interfaces = []*current.Interface{dummyInterface}
+
+	err = netns.Do(func(_ ns.NetNS) error {
+		if err := ipam.ConfigureIface(args.IfName, result); err != nil {
+			return err
+		}
+		return nil
+	})
+
+	if err != nil {
+		return err
+	}
+
+	return types.PrintResult(result, conf.CNIVersion)
+}
+
+func cmdDel(args *skel.CmdArgs) error {
+	conf, err := parseNetConf(args.StdinData)
+	if err != nil {
+		return err
+	}
+
+	if err = ipam.ExecDel(conf.IPAM.Type, args.StdinData); err != nil {
+		return err
+	}
+
+	if args.Netns == "" {
+		return nil
+	}
+
+	err = ns.WithNetNSPath(args.Netns, func(ns.NetNS) error {
+		err = ip.DelLinkByName(args.IfName)
+		if err != nil && err == ip.ErrLinkNotFound {
+			return nil
+		}
+		return err
+	})
+
+	if err != nil {
+		//  if NetNs is passed down by the Cloud Orchestration Engine, or if it called multiple times
+		// so don't return an error if the device is already removed.
+		// https://github.com/kubernetes/kubernetes/issues/43014#issuecomment-287164444
+		_, ok := err.(ns.NSPathNotExistErr)
+		if ok {
+			return nil
+		}
+		return err
+	}
+
+	return nil
+}
+
+func main() {
+	skel.PluginMain(cmdAdd, cmdCheck, cmdDel, version.All, bv.BuildString("dummy"))
+}
+
+func cmdCheck(args *skel.CmdArgs) error {
+	conf, err := parseNetConf(args.StdinData)
+	if err != nil {
+		return err
+	}
+
+	if conf.IPAM.Type == "" {
+		return errors.New("dummy interface requires an IPAM configuration")
+	}
+
+	netns, err := ns.GetNS(args.Netns)
+	if err != nil {
+		return fmt.Errorf("failed to open netns %q: %v", args.Netns, err)
+	}
+	defer netns.Close()
+
+	// run the IPAM plugin and get back the config to apply
+	err = ipam.ExecCheck(conf.IPAM.Type, args.StdinData)
+	if err != nil {
+		return err
+	}
+
+	if conf.RawPrevResult == nil {
+		return fmt.Errorf("dummy: Required prevResult missing")
+	}
+
+	if err := version.ParsePrevResult(conf); err != nil {
+		return err
+	}
+
+	// Convert whatever the IPAM result was into the current Result type
+	result, err := current.NewResultFromResult(conf.PrevResult)
+	if err != nil {
+		return err
+	}
+
+	var contMap current.Interface
+	// Find interfaces for name whe know, that of dummy device inside container
+	for _, intf := range result.Interfaces {
+		if args.IfName == intf.Name {
+			if args.Netns == intf.Sandbox {
+				contMap = *intf
+				continue
+			}
+		}
+	}
+
+	// The namespace must be the same as what was configured
+	if args.Netns != contMap.Sandbox {
+		return fmt.Errorf("Sandbox in prevResult %s doesn't match configured netns: %s",
+			contMap.Sandbox, args.Netns)
+	}
+
+	//
+	// Check prevResults for ips, routes and dns against values found in the container
+	if err := netns.Do(func(_ ns.NetNS) error {
+
+		// Check interface against values found in the container
+		err := validateCniContainerInterface(contMap)
+		if err != nil {
+			return err
+		}
+
+		err = ip.ValidateExpectedInterfaceIPs(args.IfName, result.IPs)
+		if err != nil {
+			return err
+		}
+		return nil
+	}); err != nil {
+		return err
+	}
+
+	return nil
+
+}
+
+func validateCniContainerInterface(intf current.Interface) error {
+
+	var link netlink.Link
+	var err error
+
+	if intf.Name == "" {
+		return fmt.Errorf("Container interface name missing in prevResult: %v", intf.Name)
+	}
+	link, err = netlink.LinkByName(intf.Name)
+	if err != nil {
+		return fmt.Errorf("Container Interface name in prevResult: %s not found", intf.Name)
+	}
+	if intf.Sandbox == "" {
+		return fmt.Errorf("Error: Container interface %s should not be in host namespace", link.Attrs().Name)
+	}
+
+	_, isDummy := link.(*netlink.Dummy)
+	if !isDummy {
+		return fmt.Errorf("Error: Container interface %s not of type dummy", link.Attrs().Name)
+	}
+
+	if intf.Mac != "" {
+		if intf.Mac != link.Attrs().HardwareAddr.String() {
+			return fmt.Errorf("Interface %s Mac %s doesn't match container Mac: %s", intf.Name, intf.Mac, link.Attrs().HardwareAddr)
+		}
+	}
+
+	if link.Attrs().Flags&net.FlagUp != net.FlagUp {
+		return fmt.Errorf("Interface %s is down", intf.Name)
+	}
+
+	return nil
+}

plugins/main/dummy/dummy_suite_test.go

@@ -0,0 +1,41 @@
+// Copyright 2022 Arista Networks
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main_test
+
+import (
+	"github.com/onsi/gomega/gexec"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"testing"
+)
+
+var pathToLoPlugin string
+
+func TestLoopback(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "plugins/main/dummy")
+}
+
+var _ = BeforeSuite(func() {
+	var err error
+	pathToLoPlugin, err = gexec.Build("github.com/containernetworking/plugins/plugins/main/dummy")
+	Expect(err).NotTo(HaveOccurred())
+})
+
+var _ = AfterSuite(func() {
+	gexec.CleanupBuildArtifacts()
+})

plugins/main/dummy/dummy_test.go

@@ -0,0 +1,391 @@
+// Copyright 2022 Arista Networks
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net"
+	"os"
+	"strings"
+	"syscall"
+
+	"github.com/containernetworking/cni/pkg/skel"
+	"github.com/containernetworking/cni/pkg/types"
+	types020 "github.com/containernetworking/cni/pkg/types/020"
+	types040 "github.com/containernetworking/cni/pkg/types/040"
+	types100 "github.com/containernetworking/cni/pkg/types/100"
+	"github.com/containernetworking/plugins/pkg/ns"
+	"github.com/containernetworking/plugins/pkg/testutils"
+	"github.com/containernetworking/plugins/plugins/ipam/host-local/backend/allocator"
+
+	"github.com/vishvananda/netlink"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+const MASTER_NAME = "eth0"
+
+type Net struct {
+	Name          string                 `json:"name"`
+	CNIVersion    string                 `json:"cniVersion"`
+	Type          string                 `json:"type,omitempty"`
+	IPAM          *allocator.IPAMConfig  `json:"ipam"`
+	RawPrevResult map[string]interface{} `json:"prevResult,omitempty"`
+	PrevResult    types100.Result        `json:"-"`
+}
+
+func buildOneConfig(netName string, cniVersion string, orig *Net, prevResult types.Result) (*Net, error) {
+	var err error
+
+	inject := map[string]interface{}{
+		"name":       netName,
+		"cniVersion": cniVersion,
+	}
+	// Add previous plugin result
+	if prevResult != nil {
+		inject["prevResult"] = prevResult
+	}
+
+	// Ensure every config uses the same name and version
+	config := make(map[string]interface{})
+
+	confBytes, err := json.Marshal(orig)
+	if err != nil {
+		return nil, err
+	}
+
+	err = json.Unmarshal(confBytes, &config)
+	if err != nil {
+		return nil, fmt.Errorf("unmarshal existing network bytes: %s", err)
+	}
+
+	for key, value := range inject {
+		config[key] = value
+	}
+
+	newBytes, err := json.Marshal(config)
+	if err != nil {
+		return nil, err
+	}
+
+	conf := &Net{}
+	if err := json.Unmarshal(newBytes, &conf); err != nil {
+		return nil, fmt.Errorf("error parsing configuration: %s", err)
+	}
+
+	return conf, nil
+
+}
+
+type tester interface {
+	// verifyResult minimally verifies the Result and returns the interface's MAC address
+	verifyResult(result types.Result, name string) string
+}
+
+type testerBase struct{}
+
+type testerV10x testerBase
+type testerV04x testerBase
+type testerV03x testerBase
+type testerV01xOr02x testerBase
+
+func newTesterByVersion(version string) tester {
+	switch {
+	case strings.HasPrefix(version, "1.0."):
+		return &testerV10x{}
+	case strings.HasPrefix(version, "0.4."):
+		return &testerV04x{}
+	case strings.HasPrefix(version, "0.3."):
+		return &testerV03x{}
+	default:
+		return &testerV01xOr02x{}
+	}
+}
+
+// verifyResult minimally verifies the Result and returns the interface's MAC address
+func (t *testerV10x) verifyResult(result types.Result, name string) string {
+	r, err := types100.GetResult(result)
+	Expect(err).NotTo(HaveOccurred())
+
+	Expect(len(r.Interfaces)).To(Equal(1))
+	Expect(r.Interfaces[0].Name).To(Equal(name))
+	Expect(len(r.IPs)).To(Equal(1))
+
+	return r.Interfaces[0].Mac
+}
+
+func verify0403(result types.Result, name string) string {
+	r, err := types040.GetResult(result)
+	Expect(err).NotTo(HaveOccurred())
+
+	Expect(len(r.Interfaces)).To(Equal(1))
+	Expect(r.Interfaces[0].Name).To(Equal(name))
+	Expect(len(r.IPs)).To(Equal(1))
+
+	return r.Interfaces[0].Mac
+}
+
+// verifyResult minimally verifies the Result and returns the interface's MAC address
+func (t *testerV04x) verifyResult(result types.Result, name string) string {
+	return verify0403(result, name)
+}
+
+// verifyResult minimally verifies the Result and returns the interface's MAC address
+func (t *testerV03x) verifyResult(result types.Result, name string) string {
+	return verify0403(result, name)
+}
+
+// verifyResult minimally verifies the Result and returns the interface's MAC address
+func (t *testerV01xOr02x) verifyResult(result types.Result, name string) string {
+	r, err := types020.GetResult(result)
+	Expect(err).NotTo(HaveOccurred())
+
+	Expect(r.IP4.IP.IP).NotTo(BeNil())
+	Expect(r.IP6).To(BeNil())
+
+	// 0.2 and earlier don't return MAC address
+	return ""
+}
+
+var _ = Describe("dummy Operations", func() {
+	var originalNS, targetNS ns.NetNS
+	var dataDir string
+
+	BeforeEach(func() {
+		// Create a new NetNS so we don't modify the host
+		var err error
+		originalNS, err = testutils.NewNS()
+		Expect(err).NotTo(HaveOccurred())
+		targetNS, err = testutils.NewNS()
+		Expect(err).NotTo(HaveOccurred())
+
+		dataDir, err = os.MkdirTemp("", "dummy_test")
+		Expect(err).NotTo(HaveOccurred())
+
+		err = originalNS.Do(func(ns.NetNS) error {
+			defer GinkgoRecover()
+
+			// Add master
+			err = netlink.LinkAdd(&netlink.Dummy{
+				LinkAttrs: netlink.LinkAttrs{
+					Name: MASTER_NAME,
+				},
+			})
+			Expect(err).NotTo(HaveOccurred())
+			m, err := netlink.LinkByName(MASTER_NAME)
+			Expect(err).NotTo(HaveOccurred())
+			err = netlink.LinkSetUp(m)
+			Expect(err).NotTo(HaveOccurred())
+			return nil
+		})
+		Expect(err).NotTo(HaveOccurred())
+	})
+
+	AfterEach(func() {
+		Expect(os.RemoveAll(dataDir)).To(Succeed())
+		Expect(originalNS.Close()).To(Succeed())
+		Expect(testutils.UnmountNS(originalNS)).To(Succeed())
+		Expect(targetNS.Close()).To(Succeed())
+		Expect(testutils.UnmountNS(targetNS)).To(Succeed())
+	})
+
+	for _, ver := range testutils.AllSpecVersions {
+		// Redefine ver inside for scope so real value is picked up by each dynamically defined It()
+		// See Gingkgo's "Patterns for dynamically generating tests" documentation.
+		ver := ver
+
+		It(fmt.Sprintf("[%s] creates an dummy link in a non-default namespace", ver), func() {
+			conf := &types.NetConf{
+				CNIVersion: ver,
+				Name:       "testConfig",
+				Type:       "dummy",
+			}
+
+			// Create dummy in other namespace
+			err := originalNS.Do(func(ns.NetNS) error {
+				defer GinkgoRecover()
+
+				_, err := createDummy(conf, "foobar0", targetNS)
+				Expect(err).NotTo(HaveOccurred())
+				return nil
+			})
+			Expect(err).NotTo(HaveOccurred())
+
+			// Make sure dummy link exists in the target namespace
+			err = targetNS.Do(func(ns.NetNS) error {
+				defer GinkgoRecover()
+
+				link, err := netlink.LinkByName("foobar0")
+				Expect(err).NotTo(HaveOccurred())
+				Expect(link.Attrs().Name).To(Equal("foobar0"))
+				return nil
+			})
+			Expect(err).NotTo(HaveOccurred())
+		})
+
+		It(fmt.Sprintf("[%s] configures and deconfigures a dummy link with ADD/CHECK/DEL", ver), func() {
+			const IFNAME = "dummy0"
+
+			conf := fmt.Sprintf(`{
+			    "cniVersion": "%s",
+			    "name": "dummyTestv4",
+			    "type": "dummy",
+			    "ipam": {
+					"type": "host-local",
+					"subnet": "10.1.2.0/24",
+					"dataDir": "%s"
+			    }
+			}`, ver, dataDir)
+
+			args := &skel.CmdArgs{
+				ContainerID: "contDummy",
+				Netns:       targetNS.Path(),
+				IfName:      IFNAME,
+				StdinData:   []byte(conf),
+			}
+
+			t := newTesterByVersion(ver)
+
+			var result types.Result
+			var macAddress string
+			err := originalNS.Do(func(ns.NetNS) error {
+				defer GinkgoRecover()
+
+				var err error
+				result, _, err = testutils.CmdAddWithArgs(args, func() error {
+					return cmdAdd(args)
+				})
+				Expect(err).NotTo(HaveOccurred())
+
+				macAddress = t.verifyResult(result, IFNAME)
+				return nil
+			})
+			Expect(err).NotTo(HaveOccurred())
+			// Make sure dummy link exists in the target namespace
+			err = targetNS.Do(func(ns.NetNS) error {
+				defer GinkgoRecover()
+
+				link, err := netlink.LinkByName(IFNAME)
+				Expect(err).NotTo(HaveOccurred())
+				Expect(link.Attrs().Name).To(Equal(IFNAME))
+
+				if macAddress != "" {
+					hwaddr, err := net.ParseMAC(macAddress)
+					Expect(err).NotTo(HaveOccurred())
+					Expect(link.Attrs().HardwareAddr).To(Equal(hwaddr))
+				}
+
+				addrs, err := netlink.AddrList(link, syscall.AF_INET)
+				Expect(err).NotTo(HaveOccurred())
+				Expect(len(addrs)).To(Equal(1))
+				return nil
+			})
+			Expect(err).NotTo(HaveOccurred())
+
+			// call CmdCheck
+			n := &Net{}
+			err = json.Unmarshal([]byte(conf), &n)
+			Expect(err).NotTo(HaveOccurred())
+
+			n.IPAM, _, err = allocator.LoadIPAMConfig([]byte(conf), "")
+			Expect(err).NotTo(HaveOccurred())
+
+			newConf, err := buildOneConfig("dummyTestv4", ver, n, result)
+			Expect(err).NotTo(HaveOccurred())
+
+			confString, err := json.Marshal(newConf)
+			Expect(err).NotTo(HaveOccurred())
+
+			args.StdinData = confString
+			// CNI Check dummy in the target namespace
+			err = originalNS.Do(func(ns.NetNS) error {
+				defer GinkgoRecover()
+				return testutils.CmdCheckWithArgs(args, func() error { return cmdCheck(args) })
+			})
+			if testutils.SpecVersionHasCHECK(ver) {
+				Expect(err).NotTo(HaveOccurred())
+			} else {
+				Expect(err).To(MatchError("config version does not allow CHECK"))
+			}
+
+			args.StdinData = []byte(conf)
+
+			err = originalNS.Do(func(ns.NetNS) error {
+				defer GinkgoRecover()
+
+				err = testutils.CmdDelWithArgs(args, func() error {
+					return cmdDel(args)
+				})
+				Expect(err).NotTo(HaveOccurred())
+				return nil
+			})
+			Expect(err).NotTo(HaveOccurred())
+
+			// Make sure dummy link has been deleted
+			err = targetNS.Do(func(ns.NetNS) error {
+				defer GinkgoRecover()
+
+				link, err := netlink.LinkByName(IFNAME)
+				Expect(err).To(HaveOccurred())
+				Expect(link).To(BeNil())
+				return nil
+			})
+			Expect(err).NotTo(HaveOccurred())
+
+			// DEL can be called multiple times, make sure no error is returned
+			// if the device is already removed.
+			err = originalNS.Do(func(ns.NetNS) error {
+				defer GinkgoRecover()
+
+				err = testutils.CmdDelWithArgs(args, func() error {
+					return cmdDel(args)
+				})
+				Expect(err).NotTo(HaveOccurred())
+				return nil
+			})
+			Expect(err).NotTo(HaveOccurred())
+		})
+
+		It(fmt.Sprintf("[%s] fails to create dummy link with no ipam", ver), func() {
+			var err error
+
+			const confFmt = `{
+			    "cniVersion": "%s",
+			    "name": "mynet",
+			    "type": "dummy"
+			}`
+
+			args := &skel.CmdArgs{
+				ContainerID: "dummyCont",
+				Netns:       "/var/run/netns/test",
+				IfName:      "dummy0",
+				StdinData:   []byte(fmt.Sprintf(confFmt, ver)),
+			}
+
+			_ = originalNS.Do(func(netNS ns.NetNS) error {
+				defer GinkgoRecover()
+
+				_, _, err = testutils.CmdAddWithArgs(args, func() error {
+					return cmdAdd(args)
+				})
+				Expect(err).To(Equal(errors.New("dummy interface requires an IPAM configuration")))
+				return nil
+			})
+		})
+	}
+})

plugins/main/host-device/README.md

@@ -1,5 +1,5 @@
 
 This document has moved to the [containernetworking/cni.dev](https://github.com/containernetworking/cni.dev) repo.
 
-You can find it online here: https://cni.dev/plugins/main/host-device/
+You can find it online here: https://cni.dev/plugins/current/main/host-device/
 

plugins/main/host-device/host-device.go

@@ -19,7 +19,6 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"io/ioutil"
 	"net"
 	"os"
 	"path/filepath"
@@ -30,7 +29,7 @@ import (
 
 	"github.com/containernetworking/cni/pkg/skel"
 	"github.com/containernetworking/cni/pkg/types"
-	"github.com/containernetworking/cni/pkg/types/current"
+	current "github.com/containernetworking/cni/pkg/types/100"
 	"github.com/containernetworking/cni/pkg/version"
 
 	"github.com/containernetworking/plugins/pkg/ip"
@@ -39,18 +38,19 @@ import (
 	bv "github.com/containernetworking/plugins/pkg/utils/buildversion"
 )
 
-const (
+var (
 	sysBusPCI = "/sys/bus/pci/devices"
 )
 
 // Array of different linux drivers bound to network device needed for DPDK
 var userspaceDrivers = []string{"vfio-pci", "uio_pci_generic", "igb_uio"}
 
-//NetConf for host-device config, look the README to learn how to use those parameters
+// NetConf for host-device config, look the README to learn how to use those parameters
 type NetConf struct {
 	types.NetConf
-	Device        string `json:"device"`     // Device-Name, something like eth0 or can0 etc.
-	HWAddr        string `json:"hwaddr"`     // MAC Address of target network interface
+	Device        string `json:"device"` // Device-Name, something like eth0 or can0 etc.
+	HWAddr        string `json:"hwaddr"` // MAC Address of target network interface
+	DPDKMode      bool
 	KernelPath    string `json:"kernelpath"` // Kernelpath of the device
 	PCIAddr       string `json:"pciBusID"`   // PCI Address of target network device
 	RuntimeConfig struct {
@@ -67,7 +67,8 @@ func init() {
 
 func loadConf(bytes []byte) (*NetConf, error) {
 	n := &NetConf{}
-	if err := json.Unmarshal(bytes, n); err != nil {
+	var err error
+	if err = json.Unmarshal(bytes, n); err != nil {
 		return nil, fmt.Errorf("failed to load netconf: %v", err)
 	}
 
@@ -80,6 +81,13 @@ func loadConf(bytes []byte) (*NetConf, error) {
 		return nil, fmt.Errorf(`specify either "device", "hwaddr", "kernelpath" or "pciBusID"`)
 	}
 
+	if len(n.PCIAddr) > 0 {
+		n.DPDKMode, err = hasDpdkDriver(n.PCIAddr)
+		if err != nil {
+			return nil, fmt.Errorf("error with host device: %v", err)
+		}
+	}
+
 	return n, nil
 }
 
@@ -94,49 +102,17 @@ func cmdAdd(args *skel.CmdArgs) error {
 	}
 	defer containerNs.Close()
 
-	if len(cfg.PCIAddr) > 0 {
-		isDpdkMode, err := hasDpdkDriver(cfg.PCIAddr)
+	result := &current.Result{}
+	var contDev netlink.Link
+	if !cfg.DPDKMode {
+		hostDev, err := getLink(cfg.Device, cfg.HWAddr, cfg.KernelPath, cfg.PCIAddr)
 		if err != nil {
-			return fmt.Errorf("error with host device: %v", err)
+			return fmt.Errorf("failed to find host device: %v", err)
 		}
-		if isDpdkMode {
-			return types.PrintResult(&current.Result{}, cfg.CNIVersion)
-		}
-	}
 
-	hostDev, err := getLink(cfg.Device, cfg.HWAddr, cfg.KernelPath, cfg.PCIAddr)
-	if err != nil {
-		return fmt.Errorf("failed to find host device: %v", err)
-	}
-
-	contDev, err := moveLinkIn(hostDev, containerNs, args.IfName)
-	if err != nil {
-		return fmt.Errorf("failed to move link %v", err)
-	}
-
-	var result *current.Result
-	// run the IPAM plugin and get back the config to apply
-	if cfg.IPAM.Type != "" {
-		r, err := ipam.ExecAdd(cfg.IPAM.Type, args.StdinData)
+		contDev, err = moveLinkIn(hostDev, containerNs, args.IfName)
 		if err != nil {
-			return err
-		}
-
-		// Invoke ipam del if err to avoid ip leak
-		defer func() {
-			if err != nil {
-				ipam.ExecDel(cfg.IPAM.Type, args.StdinData)
-			}
-		}()
-
-		// Convert whatever the IPAM result was into the current Result type
-		result, err = current.NewResultFromResult(r)
-		if err != nil {
-			return err
-		}
-
-		if len(result.IPs) == 0 {
-			return errors.New("IPAM plugin returned missing IP config")
+			return fmt.Errorf("failed to move link %v", err)
 		}
 
 		result.Interfaces = []*current.Interface{{
@@ -144,13 +120,48 @@ func cmdAdd(args *skel.CmdArgs) error {
 			Mac:     contDev.Attrs().HardwareAddr.String(),
 			Sandbox: containerNs.Path(),
 		}}
-		for _, ipc := range result.IPs {
-			// All addresses apply to the container interface (move from host)
-			ipc.Interface = current.Int(0)
-		}
+	}
 
+	if cfg.IPAM.Type == "" {
+		if cfg.DPDKMode {
+			return types.PrintResult(result, cfg.CNIVersion)
+		}
+		return printLink(contDev, cfg.CNIVersion, containerNs)
+	}
+
+	// run the IPAM plugin and get back the config to apply
+	r, err := ipam.ExecAdd(cfg.IPAM.Type, args.StdinData)
+	if err != nil {
+		return err
+	}
+
+	// Invoke ipam del if err to avoid ip leak
+	defer func() {
+		if err != nil {
+			ipam.ExecDel(cfg.IPAM.Type, args.StdinData)
+		}
+	}()
+
+	// Convert whatever the IPAM result was into the current Result type
+	newResult, err := current.NewResultFromResult(r)
+	if err != nil {
+		return err
+	}
+
+	if len(newResult.IPs) == 0 {
+		return errors.New("IPAM plugin returned missing IP config")
+	}
+
+	for _, ipc := range newResult.IPs {
+		// All addresses apply to the container interface (move from host)
+		ipc.Interface = current.Int(0)
+	}
+
+	newResult.Interfaces = result.Interfaces
+
+	if !cfg.DPDKMode {
 		err = containerNs.Do(func(_ ns.NetNS) error {
-			if err := ipam.ConfigureIface(args.IfName, result); err != nil {
+			if err := ipam.ConfigureIface(args.IfName, newResult); err != nil {
 				return err
 			}
 			return nil
@@ -158,13 +169,11 @@ func cmdAdd(args *skel.CmdArgs) error {
 		if err != nil {
 			return err
 		}
-
-		result.DNS = cfg.DNS
-
-		return types.PrintResult(result, cfg.CNIVersion)
 	}
 
-	return printLink(contDev, cfg.CNIVersion, containerNs)
+	newResult.DNS = cfg.DNS
+
+	return types.PrintResult(newResult, cfg.CNIVersion)
 }
 
 func cmdDel(args *skel.CmdArgs) error {
@@ -181,26 +190,18 @@ func cmdDel(args *skel.CmdArgs) error {
 	}
 	defer containerNs.Close()
 
-	if len(cfg.PCIAddr) > 0 {
-		isDpdkMode, err := hasDpdkDriver(cfg.PCIAddr)
-		if err != nil {
-			return fmt.Errorf("error with host device: %v", err)
-		}
-		if isDpdkMode {
-			return nil
-		}
-	}
-
-	if err := moveLinkOut(containerNs, args.IfName); err != nil {
-		return err
-	}
-
 	if cfg.IPAM.Type != "" {
 		if err := ipam.ExecDel(cfg.IPAM.Type, args.StdinData); err != nil {
 			return err
 		}
 	}
 
+	if !cfg.DPDKMode {
+		if err := moveLinkOut(containerNs, args.IfName); err != nil {
+			return err
+		}
+	}
+
 	return nil
 }
 
@@ -228,6 +229,10 @@ func moveLinkIn(hostDev netlink.Link, containerNs ns.NetNS, ifName string) (netl
 		if err := netlink.LinkSetName(contDev, ifName); err != nil {
 			return fmt.Errorf("failed to rename device %q to %q: %v", hostDev.Attrs().Name, ifName, err)
 		}
+		// Bring container device up
+		if err = netlink.LinkSetUp(contDev); err != nil {
+			return fmt.Errorf("failed to set %q up: %v", ifName, err)
+		}
 		// Retrieve link again to get up-to-date name and attributes
 		contDev, err = netlink.LinkByName(ifName)
 		if err != nil {
@@ -259,17 +264,21 @@ func moveLinkOut(containerNs ns.NetNS, ifName string) error {
 			return fmt.Errorf("failed to set %q down: %v", ifName, err)
 		}
 
-		// Rename device to it's original name
+		defer func() {
+			// If moving the device to the host namespace fails, set its name back to ifName so that this
+			// function can be retried. Also bring the device back up, unless it was already down before.
+			if err != nil {
+				_ = netlink.LinkSetName(dev, ifName)
+				if dev.Attrs().Flags&net.FlagUp == net.FlagUp {
+					_ = netlink.LinkSetUp(dev)
+				}
+			}
+		}()
+
+		// Rename the device to its original name from the host namespace
 		if err = netlink.LinkSetName(dev, dev.Attrs().Alias); err != nil {
 			return fmt.Errorf("failed to restore %q to original name %q: %v", ifName, dev.Attrs().Alias, err)
 		}
-		defer func() {
-			if err != nil {
-				// if moving device to host namespace fails, we should revert device name
-				// to ifName to make sure that device can be found in retries
-				_ = netlink.LinkSetName(dev, ifName)
-			}
-		}()
 
 		if err = netlink.LinkSetNsFd(dev, int(defaultNs.Fd())); err != nil {
 			return fmt.Errorf("failed to move %q to host netns: %v", dev.Attrs().Alias, err)
@@ -335,16 +344,16 @@ func getLink(devname, hwaddr, kernelpath, pciaddr string) (netlink.Link, error)
 			return nil, fmt.Errorf("kernel device path %q must be absolute and begin with /sys/devices/", kernelpath)
 		}
 		netDir := filepath.Join(kernelpath, "net")
-		files, err := ioutil.ReadDir(netDir)
+		entries, err := os.ReadDir(netDir)
 		if err != nil {
 			return nil, fmt.Errorf("failed to find network devices at %q", netDir)
 		}
 
 		// Grab the first device from eg /sys/devices/pci0000:00/0000:00:19.0/net
-		for _, file := range files {
+		for _, entry := range entries {
 			// Make sure it's really an interface
 			for _, l := range links {
-				if file.Name() == l.Attrs().Name {
+				if entry.Name() == l.Attrs().Name {
 					return l, nil
 				}
 			}
@@ -359,12 +368,12 @@ func getLink(devname, hwaddr, kernelpath, pciaddr string) (netlink.Link, error)
 			}
 			netDir = matches[0]
 		}
-		fInfo, err := ioutil.ReadDir(netDir)
+		entries, err := os.ReadDir(netDir)
 		if err != nil {
 			return nil, fmt.Errorf("failed to read net directory %s: %q", netDir, err)
 		}
-		if len(fInfo) > 0 {
-			return netlink.LinkByName(fInfo[0].Name())
+		if len(entries) > 0 {
+			return netlink.LinkByName(entries[0].Name())
 		}
 		return nil, fmt.Errorf("failed to find device name for pci address %s", pciaddr)
 	}
@@ -410,6 +419,10 @@ func cmdCheck(args *skel.CmdArgs) error {
 		return err
 	}
 
+	if cfg.DPDKMode {
+		return nil
+	}
+
 	var contMap current.Interface
 	// Find interfaces for name we know, that of host-device inside container
 	for _, intf := range result.Interfaces {

plugins/main/ipvlan/README.md

@@ -1,5 +1,5 @@
 
 This document has moved to the [containernetworking/cni.dev](https://github.com/containernetworking/cni.dev) repo.
 
-You can find it online here: https://cni.dev/plugins/main/ipvlan/
+You can find it online here: https://cni.dev/plugins/current/main/ipvlan/
 

plugins/main/ipvlan/ipvlan.go

@@ -24,13 +24,14 @@ import (
 
 	"github.com/containernetworking/cni/pkg/skel"
 	"github.com/containernetworking/cni/pkg/types"
-	"github.com/containernetworking/cni/pkg/types/current"
+	current "github.com/containernetworking/cni/pkg/types/100"
 	"github.com/containernetworking/cni/pkg/version"
 
 	"github.com/containernetworking/plugins/pkg/ip"
 	"github.com/containernetworking/plugins/pkg/ipam"
 	"github.com/containernetworking/plugins/pkg/ns"
 	bv "github.com/containernetworking/plugins/pkg/utils/buildversion"
+	"github.com/containernetworking/plugins/pkg/utils/sysctl"
 )
 
 type NetConf struct {
@@ -254,7 +255,12 @@ func cmdAdd(args *skel.CmdArgs) error {
 	result.Interfaces = []*current.Interface{ipvlanInterface}
 
 	err = netns.Do(func(_ ns.NetNS) error {
-		return ipam.ConfigureIface(args.IfName, result)
+		_, _ = sysctl.Sysctl(fmt.Sprintf("net/ipv4/conf/%s/arp_notify", args.IfName), "1")
+
+		if err := ipam.ConfigureIface(args.IfName, result); err != nil {
+			return err
+		}
+		return nil
 	})
 	if err != nil {
 		return err
@@ -294,6 +300,17 @@ func cmdDel(args *skel.CmdArgs) error {
 		return nil
 	})
 
+	if err != nil {
+		//  if NetNs is passed down by the Cloud Orchestration Engine, or if it called multiple times
+		// so don't return an error if the device is already removed.
+		// https://github.com/kubernetes/kubernetes/issues/43014#issuecomment-287164444
+		_, ok := err.(ns.NSPathNotExistErr)
+		if ok {
+			return nil
+		}
+		return err
+	}
+
 	return err
 }
 

plugins/main/ipvlan/ipvlan_test.go

@@ -18,11 +18,15 @@ import (
 	"encoding/json"
 	"fmt"
 	"net"
+	"os"
+	"strings"
 	"syscall"
 
 	"github.com/containernetworking/cni/pkg/skel"
 	"github.com/containernetworking/cni/pkg/types"
-	"github.com/containernetworking/cni/pkg/types/current"
+	types020 "github.com/containernetworking/cni/pkg/types/020"
+	types040 "github.com/containernetworking/cni/pkg/types/040"
+	types100 "github.com/containernetworking/cni/pkg/types/100"
 	"github.com/containernetworking/plugins/pkg/ns"
 	"github.com/containernetworking/plugins/pkg/testutils"
 
@@ -44,37 +48,33 @@ type Net struct {
 	IPAM          *allocator.IPAMConfig  `json:"ipam"`
 	DNS           types.DNS              `json:"dns"`
 	RawPrevResult map[string]interface{} `json:"prevResult,omitempty"`
-	PrevResult    current.Result         `json:"-"`
+	PrevResult    types100.Result        `json:"-"`
 }
 
-func buildOneConfig(netName string, cniVersion string, master string, orig *Net, prevResult types.Result) (*Net, error) {
-	var err error
-
-	inject := map[string]interface{}{
-		"name":       netName,
-		"cniVersion": cniVersion,
-	}
-	// Add previous plugin result
-	if prevResult != nil {
-		inject["prevResult"] = prevResult
-	}
-	if orig.IPAM == nil {
-		inject["master"] = master
-	}
-
-	// Ensure every config uses the same name and version
-	config := make(map[string]interface{})
-
+func buildOneConfig(cniVersion string, master string, orig *Net, prevResult types.Result) (*Net, error) {
 	confBytes, err := json.Marshal(orig)
 	if err != nil {
 		return nil, err
 	}
 
+	config := make(map[string]interface{})
 	err = json.Unmarshal(confBytes, &config)
 	if err != nil {
 		return nil, fmt.Errorf("unmarshal existing network bytes: %s", err)
 	}
 
+	inject := map[string]interface{}{
+		"name":       orig.Name,
+		"cniVersion": orig.CNIVersion,
+	}
+	// Add previous plugin result
+	if prevResult != nil && testutils.SpecVersionHasChaining(cniVersion) {
+		inject["prevResult"] = prevResult
+	}
+	if master != "" {
+		inject["master"] = master
+	}
+
 	for key, value := range inject {
 		config[key] = value
 	}
@@ -93,121 +93,49 @@ func buildOneConfig(netName string, cniVersion string, master string, orig *Net,
 
 }
 
-func ipvlanAddDelTest(conf, IFNAME string, originalNS ns.NetNS) {
-	targetNs, err := testutils.NewNS()
+func ipvlanAddCheckDelTest(conf, masterName string, originalNS, targetNS ns.NetNS) {
+	// Unmarshal to pull out CNI spec version
+	rawConfig := make(map[string]interface{})
+	err := json.Unmarshal([]byte(conf), &rawConfig)
 	Expect(err).NotTo(HaveOccurred())
-	defer targetNs.Close()
+	cniVersion := rawConfig["cniVersion"].(string)
 
 	args := &skel.CmdArgs{
 		ContainerID: "dummy",
-		Netns:       targetNs.Path(),
-		IfName:      IFNAME,
+		Netns:       targetNS.Path(),
+		IfName:      "ipvl0",
 		StdinData:   []byte(conf),
 	}
 
-	var result *current.Result
+	var result types.Result
+	var macAddress string
 	err = originalNS.Do(func(ns.NetNS) error {
 		defer GinkgoRecover()
 
-		r, _, err := testutils.CmdAddWithArgs(args, func() error {
+		result, _, err = testutils.CmdAddWithArgs(args, func() error {
 			return cmdAdd(args)
 		})
 		Expect(err).NotTo(HaveOccurred())
 
-		result, err = current.GetResult(r)
-		Expect(err).NotTo(HaveOccurred())
-
-		Expect(len(result.Interfaces)).To(Equal(1))
-		Expect(result.Interfaces[0].Name).To(Equal(IFNAME))
-		Expect(len(result.IPs)).To(Equal(1))
+		t := newTesterByVersion(cniVersion)
+		macAddress = t.verifyResult(result, args.IfName)
 		return nil
 	})
 	Expect(err).NotTo(HaveOccurred())
 
 	// Make sure ipvlan link exists in the target namespace
-	err = targetNs.Do(func(ns.NetNS) error {
+	err = targetNS.Do(func(ns.NetNS) error {
 		defer GinkgoRecover()
 
-		link, err := netlink.LinkByName(IFNAME)
+		link, err := netlink.LinkByName(args.IfName)
 		Expect(err).NotTo(HaveOccurred())
-		Expect(link.Attrs().Name).To(Equal(IFNAME))
+		Expect(link.Attrs().Name).To(Equal(args.IfName))
 
-		hwaddr, err := net.ParseMAC(result.Interfaces[0].Mac)
-		Expect(err).NotTo(HaveOccurred())
-		Expect(link.Attrs().HardwareAddr).To(Equal(hwaddr))
-
-		addrs, err := netlink.AddrList(link, syscall.AF_INET)
-		Expect(err).NotTo(HaveOccurred())
-		Expect(len(addrs)).To(Equal(1))
-		return nil
-	})
-	Expect(err).NotTo(HaveOccurred())
-
-	err = originalNS.Do(func(ns.NetNS) error {
-		defer GinkgoRecover()
-
-		err = testutils.CmdDelWithArgs(args, func() error {
-			return cmdDel(args)
-		})
-		Expect(err).NotTo(HaveOccurred())
-		return nil
-	})
-	Expect(err).NotTo(HaveOccurred())
-
-	// Make sure ipvlan link has been deleted
-	err = targetNs.Do(func(ns.NetNS) error {
-		defer GinkgoRecover()
-
-		link, err := netlink.LinkByName(IFNAME)
-		Expect(err).To(HaveOccurred())
-		Expect(link).To(BeNil())
-		return nil
-	})
-	Expect(err).NotTo(HaveOccurred())
-}
-
-func ipvlanAddCheckDelTest(conf string, netName string, IFNAME string, originalNS ns.NetNS) {
-	targetNs, err := testutils.NewNS()
-	Expect(err).NotTo(HaveOccurred())
-	defer targetNs.Close()
-
-	args := &skel.CmdArgs{
-		ContainerID: "dummy",
-		Netns:       targetNs.Path(),
-		IfName:      IFNAME,
-		StdinData:   []byte(conf),
-	}
-
-	var result *current.Result
-	err = originalNS.Do(func(ns.NetNS) error {
-		defer GinkgoRecover()
-
-		r, _, err := testutils.CmdAddWithArgs(args, func() error {
-			return cmdAdd(args)
-		})
-		Expect(err).NotTo(HaveOccurred())
-
-		result, err = current.GetResult(r)
-		Expect(err).NotTo(HaveOccurred())
-
-		Expect(len(result.Interfaces)).To(Equal(1))
-		Expect(result.Interfaces[0].Name).To(Equal(IFNAME))
-		Expect(len(result.IPs)).To(Equal(1))
-		return nil
-	})
-	Expect(err).NotTo(HaveOccurred())
-
-	// Make sure ipvlan link exists in the target namespace
-	err = targetNs.Do(func(ns.NetNS) error {
-		defer GinkgoRecover()
-
-		link, err := netlink.LinkByName(IFNAME)
-		Expect(err).NotTo(HaveOccurred())
-		Expect(link.Attrs().Name).To(Equal(IFNAME))
-
-		hwaddr, err := net.ParseMAC(result.Interfaces[0].Mac)
-		Expect(err).NotTo(HaveOccurred())
-		Expect(link.Attrs().HardwareAddr).To(Equal(hwaddr))
+		if macAddress != "" {
+			hwaddr, err := net.ParseMAC(macAddress)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(link.Attrs().HardwareAddr).To(Equal(hwaddr))
+		}
 
 		addrs, err := netlink.AddrList(link, syscall.AF_INET)
 		Expect(err).NotTo(HaveOccurred())
@@ -225,26 +153,26 @@ func ipvlanAddCheckDelTest(conf string, netName string, IFNAME string, originalN
 		Expect(err).NotTo(HaveOccurred())
 	}
 
-	cniVersion := "0.4.0"
-	newConf, err := buildOneConfig(netName, cniVersion, MASTER_NAME, n, result)
+	// build chained/cached config for DEL
+	newConf, err := buildOneConfig(cniVersion, masterName, n, result)
+	Expect(err).NotTo(HaveOccurred())
+	confBytes, err := json.Marshal(newConf)
 	Expect(err).NotTo(HaveOccurred())
 
-	confString, err := json.Marshal(newConf)
-	Expect(err).NotTo(HaveOccurred())
+	args.StdinData = confBytes
+	GinkgoT().Logf(string(confBytes))
 
-	args.StdinData = confString
+	if testutils.SpecVersionHasCHECK(cniVersion) {
+		// CNI Check on ipvlan in the target namespace
+		err = originalNS.Do(func(ns.NetNS) error {
+			defer GinkgoRecover()
 
-	// CNI Check on ipvlan in the target namespace
-	err = originalNS.Do(func(ns.NetNS) error {
-		defer GinkgoRecover()
-
-		err := testutils.CmdCheckWithArgs(args, func() error {
-			return cmdCheck(args)
+			return testutils.CmdCheckWithArgs(args, func() error {
+				return cmdCheck(args)
+			})
 		})
 		Expect(err).NotTo(HaveOccurred())
-		return nil
-	})
-	Expect(err).NotTo(HaveOccurred())
+	}
 
 	err = originalNS.Do(func(ns.NetNS) error {
 		defer GinkgoRecover()
@@ -258,10 +186,10 @@ func ipvlanAddCheckDelTest(conf string, netName string, IFNAME string, originalN
 	Expect(err).NotTo(HaveOccurred())
 
 	// Make sure ipvlan link has been deleted
-	err = targetNs.Do(func(ns.NetNS) error {
+	err = targetNS.Do(func(ns.NetNS) error {
 		defer GinkgoRecover()
 
-		link, err := netlink.LinkByName(IFNAME)
+		link, err := netlink.LinkByName(args.IfName)
 		Expect(err).To(HaveOccurred())
 		Expect(link).To(BeNil())
 		return nil
@@ -269,8 +197,70 @@ func ipvlanAddCheckDelTest(conf string, netName string, IFNAME string, originalN
 	Expect(err).NotTo(HaveOccurred())
 }
 
+type tester interface {
+	// verifyResult minimally verifies the Result and returns the interface's MAC address
+	verifyResult(result types.Result, name string) string
+}
+
+type testerBase struct{}
+
+type testerV10x testerBase
+type testerV04x testerBase
+type testerV02x testerBase
+
+func newTesterByVersion(version string) tester {
+	switch {
+	case strings.HasPrefix(version, "1.0."):
+		return &testerV10x{}
+	case strings.HasPrefix(version, "0.4.") || strings.HasPrefix(version, "0.3."):
+		return &testerV04x{}
+	case strings.HasPrefix(version, "0.1.") || strings.HasPrefix(version, "0.2."):
+		return &testerV02x{}
+	}
+	Fail(fmt.Sprintf("unsupported config version %s", version))
+	return nil
+}
+
+// verifyResult minimally verifies the Result and returns the interface's MAC address
+func (t *testerV10x) verifyResult(result types.Result, name string) string {
+	r, err := types100.GetResult(result)
+	Expect(err).NotTo(HaveOccurred())
+
+	Expect(len(r.Interfaces)).To(Equal(1))
+	Expect(r.Interfaces[0].Name).To(Equal(name))
+	Expect(len(r.IPs)).To(Equal(1))
+
+	return r.Interfaces[0].Mac
+}
+
+// verifyResult minimally verifies the Result and returns the interface's MAC address
+func (t *testerV04x) verifyResult(result types.Result, name string) string {
+	r, err := types040.GetResult(result)
+	Expect(err).NotTo(HaveOccurred())
+
+	Expect(len(r.Interfaces)).To(Equal(1))
+	Expect(r.Interfaces[0].Name).To(Equal(name))
+	Expect(len(r.IPs)).To(Equal(1))
+
+	return r.Interfaces[0].Mac
+}
+
+// verifyResult minimally verifies the Result and returns the interface's MAC address
+func (t *testerV02x) verifyResult(result types.Result, name string) string {
+	r, err := types020.GetResult(result)
+	Expect(err).NotTo(HaveOccurred())
+
+	Expect(r.IP4.IP).NotTo(BeNil())
+	Expect(r.IP4.IP.IP).NotTo(BeNil())
+	Expect(r.IP6).To(BeNil())
+
+	// 0.2 and earlier don't return MAC address
+	return ""
+}
+
 var _ = Describe("ipvlan Operations", func() {
-	var originalNS ns.NetNS
+	var originalNS, targetNS ns.NetNS
+	var dataDir string
 
 	BeforeEach(func() {
 		// Create a new NetNS so we don't modify the host
@@ -278,6 +268,12 @@ var _ = Describe("ipvlan Operations", func() {
 		originalNS, err = testutils.NewNS()
 		Expect(err).NotTo(HaveOccurred())
 
+		targetNS, err = testutils.NewNS()
+		Expect(err).NotTo(HaveOccurred())
+
+		dataDir, err = os.MkdirTemp("", "ipvlan_test")
+		Expect(err).NotTo(HaveOccurred())
+
 		err = originalNS.Do(func(ns.NetNS) error {
 			defer GinkgoRecover()
 
@@ -296,219 +292,170 @@ var _ = Describe("ipvlan Operations", func() {
 	})
 
 	AfterEach(func() {
+		Expect(os.RemoveAll(dataDir)).To(Succeed())
+
 		Expect(originalNS.Close()).To(Succeed())
 		Expect(testutils.UnmountNS(originalNS)).To(Succeed())
+
+		Expect(targetNS.Close()).To(Succeed())
+		Expect(testutils.UnmountNS(targetNS)).To(Succeed())
 	})
 
-	It("creates an ipvlan link in a non-default namespace", func() {
-		conf := &NetConf{
-			NetConf: types.NetConf{
-				CNIVersion: "0.3.1",
-				Name:       "testConfig",
-				Type:       "ipvlan",
-			},
-			Master: MASTER_NAME,
-			Mode:   "l2",
-			MTU:    1500,
-		}
+	for _, ver := range testutils.AllSpecVersions {
+		// Redefine ver inside for scope so real value is picked up by each dynamically defined It()
+		// See Gingkgo's "Patterns for dynamically generating tests" documentation.
+		ver := ver
 
-		// Create ipvlan in other namespace
-		targetNs, err := testutils.NewNS()
-		Expect(err).NotTo(HaveOccurred())
-		defer targetNs.Close()
+		It(fmt.Sprintf("[%s] creates an ipvlan link in a non-default namespace", ver), func() {
+			conf := &NetConf{
+				NetConf: types.NetConf{
+					CNIVersion: ver,
+					Name:       "testConfig",
+					Type:       "ipvlan",
+				},
+				Master: MASTER_NAME,
+				Mode:   "l2",
+				MTU:    1500,
+			}
 
-		err = originalNS.Do(func(ns.NetNS) error {
-			defer GinkgoRecover()
+			err := originalNS.Do(func(ns.NetNS) error {
+				defer GinkgoRecover()
+
+				_, err := createIpvlan(conf, "foobar0", targetNS)
+				Expect(err).NotTo(HaveOccurred())
+				return nil
+			})
 
-			_, err := createIpvlan(conf, "foobar0", targetNs)
 			Expect(err).NotTo(HaveOccurred())
-			return nil
-		})
 
-		Expect(err).NotTo(HaveOccurred())
+			// Make sure ipvlan link exists in the target namespace
+			err = targetNS.Do(func(ns.NetNS) error {
+				defer GinkgoRecover()
 
-		// Make sure ipvlan link exists in the target namespace
-		err = targetNs.Do(func(ns.NetNS) error {
-			defer GinkgoRecover()
-
-			link, err := netlink.LinkByName("foobar0")
-			Expect(err).NotTo(HaveOccurred())
-			Expect(link.Attrs().Name).To(Equal("foobar0"))
-			return nil
-		})
-		Expect(err).NotTo(HaveOccurred())
-	})
-
-	It("configures and deconfigures an iplvan link with ADD/DEL", func() {
-		const IFNAME = "ipvl0"
-
-		conf := fmt.Sprintf(`{
-    "cniVersion": "0.3.1",
-    "name": "mynet",
-    "type": "ipvlan",
-    "master": "%s",
-    "ipam": {
-        "type": "host-local",
-        "subnet": "10.1.2.0/24"
-    }
-}`, MASTER_NAME)
-
-		ipvlanAddDelTest(conf, IFNAME, originalNS)
-	})
-
-	It("configures and deconfigures an iplvan link with ADD/DEL when chained", func() {
-		const IFNAME = "ipvl0"
-
-		conf := fmt.Sprintf(`{
-    "cniVersion": "0.3.1",
-    "name": "mynet",
-    "type": "ipvlan",
-    "prevResult": {
-            "interfaces": [
-                    {
-                            "name": "%s"
-                    }
-            ],
-            "ips": [
-                    {
-                            "version": "4",
-                            "address": "10.1.2.2/24",
-                            "gateway": "10.1.2.1",
-                            "interface": 0
-                    }
-            ],
-            "routes": []
-    }
-}`, MASTER_NAME)
-
-		ipvlanAddDelTest(conf, IFNAME, originalNS)
-	})
-
-	It("deconfigures an unconfigured ipvlan link with DEL", func() {
-		const IFNAME = "ipvl0"
-
-		conf := fmt.Sprintf(`{
-    "cniVersion": "0.3.0",
-    "name": "mynet",
-    "type": "ipvlan",
-    "master": "%s",
-    "ipam": {
-        "type": "host-local",
-        "subnet": "10.1.2.0/24"
-    }
-}`, MASTER_NAME)
-
-		targetNs, err := testutils.NewNS()
-		Expect(err).NotTo(HaveOccurred())
-		defer targetNs.Close()
-
-		args := &skel.CmdArgs{
-			ContainerID: "dummy",
-			Netns:       targetNs.Path(),
-			IfName:      IFNAME,
-			StdinData:   []byte(conf),
-		}
-
-		err = originalNS.Do(func(ns.NetNS) error {
-			defer GinkgoRecover()
-
-			err = testutils.CmdDelWithArgs(args, func() error {
-				return cmdDel(args)
+				link, err := netlink.LinkByName("foobar0")
+				Expect(err).NotTo(HaveOccurred())
+				Expect(link.Attrs().Name).To(Equal("foobar0"))
+				return nil
 			})
 			Expect(err).NotTo(HaveOccurred())
-			return nil
 		})
-		Expect(err).NotTo(HaveOccurred())
-	})
 
-	It("configures and deconfigures a cniVersion 0.4.0 iplvan link with ADD/CHECK/DEL", func() {
-		const IFNAME = "ipvl0"
+		It(fmt.Sprintf("[%s] configures and deconfigures an iplvan link with ADD/DEL", ver), func() {
+			conf := fmt.Sprintf(`{
+			    "cniVersion": "%s",
+			    "name": "mynet",
+			    "type": "ipvlan",
+			    "master": "%s",
+			    "ipam": {
+				"type": "host-local",
+				"subnet": "10.1.2.0/24",
+				"dataDir": "%s"
+			    }
+			}`, ver, MASTER_NAME, dataDir)
 
-		conf := fmt.Sprintf(`{
-    "cniVersion": "0.4.0",
-    "name": "ipvlanTest1",
-    "type": "ipvlan",
-    "master": "%s",
-    "ipam": {
-        "type": "host-local",
-        "subnet": "10.1.2.0/24"
-    }
-}`, MASTER_NAME)
+			ipvlanAddCheckDelTest(conf, "", originalNS, targetNS)
+		})
 
-		ipvlanAddCheckDelTest(conf, "ipvlanTest1", IFNAME, originalNS)
-	})
+		if testutils.SpecVersionHasChaining(ver) {
+			It(fmt.Sprintf("[%s] configures and deconfigures an iplvan link with ADD/DEL when chained", ver), func() {
+				conf := fmt.Sprintf(`{
+				    "cniVersion": "%s",
+				    "name": "mynet",
+				    "type": "ipvlan",
+				    "prevResult": {
+					    "interfaces": [
+						    {
+							    "name": "%s"
+						    }
+					    ],
+					    "ips": [
+						    {
+							    "version": "4",
+							    "address": "10.1.2.2/24",
+							    "gateway": "10.1.2.1",
+							    "interface": 0
+						    }
+					    ],
+					    "routes": []
+				    }
+				}`, ver, MASTER_NAME)
 
-	It("configures and deconfigures a cniVersion 0.4.0 iplvan link with ADD/CHECK/DEL when chained", func() {
-		const IFNAME = "ipvl0"
+				ipvlanAddCheckDelTest(conf, MASTER_NAME, originalNS, targetNS)
+			})
+		}
 
-		conf := fmt.Sprintf(`{
-	       "cniVersion": "0.4.0",
-	       "name": "ipvlanTest2",
-	       "type": "ipvlan",
-	       "prevResult": {
-	               "interfaces": [
-	                       {
-	                               "name": "%s"
-	                       }
-	               ],
-	               "ips": [
-	                       {
-	                               "version": "4",
-	                               "address": "10.1.2.2/24",
-	                               "gateway": "10.1.2.1",
-	                               "interface": 0
-	                       }
-	               ],
-	               "routes": []
-	       }
-	   }`, MASTER_NAME)
+		It(fmt.Sprintf("[%s] deconfigures an unconfigured ipvlan link with DEL", ver), func() {
+			conf := fmt.Sprintf(`{
+			    "cniVersion": "%s",
+			    "name": "mynet",
+			    "type": "ipvlan",
+			    "master": "%s",
+			    "ipam": {
+				"type": "host-local",
+				"subnet": "10.1.2.0/24",
+				"dataDir": "%s"
+			    }
+			}`, ver, MASTER_NAME, dataDir)
 
-		ipvlanAddCheckDelTest(conf, "ipvlanTest2", IFNAME, originalNS)
-	})
-
-	It("configures and deconfigures a ipvlan link with ADD/DEL, without master config", func() {
-		const IFNAME = "ipvl0"
-		conf := `{
-    "cniVersion": "0.3.1",
-    "name": "mynet",
-    "type": "ipvlan",
-    "ipam": {
-        "type": "host-local",
-        "subnet": "10.1.2.0/24"
-    }
-}`
-
-		targetNs, err := testutils.NewNS()
-		Expect(err).NotTo(HaveOccurred())
-		defer targetNs.Close()
-
-		// Make MASTER_NAME as default route interface
-		err = originalNS.Do(func(ns.NetNS) error {
-			defer GinkgoRecover()
-
-			link, err := netlink.LinkByName(MASTER_NAME)
-			Expect(err).NotTo(HaveOccurred())
-			err = netlink.LinkSetUp(link)
-			Expect(err).NotTo(HaveOccurred())
-
-			var address = &net.IPNet{IP: net.IPv4(192, 0, 0, 1), Mask: net.CIDRMask(24, 32)}
-			var addr = &netlink.Addr{IPNet: address}
-			err = netlink.AddrAdd(link, addr)
-			Expect(err).NotTo(HaveOccurred())
-
-			// add default gateway into MASTER
-			dst := &net.IPNet{
-				IP:   net.IPv4(0, 0, 0, 0),
-				Mask: net.CIDRMask(0, 0),
+			args := &skel.CmdArgs{
+				ContainerID: "dummy",
+				Netns:       targetNS.Path(),
+				IfName:      "ipvl0",
+				StdinData:   []byte(conf),
 			}
-			ip := net.IPv4(192, 0, 0, 254)
-			route := netlink.Route{LinkIndex: link.Attrs().Index, Dst: dst, Gw: ip}
-			err = netlink.RouteAdd(&route)
-			Expect(err).NotTo(HaveOccurred())
-			return nil
-		})
-		Expect(err).NotTo(HaveOccurred())
 
-		ipvlanAddDelTest(conf, IFNAME, originalNS)
-	})
+			err := originalNS.Do(func(ns.NetNS) error {
+				defer GinkgoRecover()
+
+				err := testutils.CmdDelWithArgs(args, func() error {
+					return cmdDel(args)
+				})
+				Expect(err).NotTo(HaveOccurred())
+				return nil
+			})
+			Expect(err).NotTo(HaveOccurred())
+		})
+
+		It(fmt.Sprintf("[%s] configures and deconfigures a ipvlan link with ADD/DEL, without master config", ver), func() {
+			conf := fmt.Sprintf(`{
+			    "cniVersion": "%s",
+			    "name": "mynet",
+			    "type": "ipvlan",
+			    "ipam": {
+				"type": "host-local",
+				"subnet": "10.1.2.0/24",
+				"dataDir": "%s"
+			    }
+			}`, ver, dataDir)
+
+			// Make MASTER_NAME as default route interface
+			err := originalNS.Do(func(ns.NetNS) error {
+				defer GinkgoRecover()
+
+				link, err := netlink.LinkByName(MASTER_NAME)
+				Expect(err).NotTo(HaveOccurred())
+				err = netlink.LinkSetUp(link)
+				Expect(err).NotTo(HaveOccurred())
+
+				var address = &net.IPNet{IP: net.IPv4(192, 0, 0, 1), Mask: net.CIDRMask(24, 32)}
+				var addr = &netlink.Addr{IPNet: address}
+				err = netlink.AddrAdd(link, addr)
+				Expect(err).NotTo(HaveOccurred())
+
+				// add default gateway into MASTER
+				dst := &net.IPNet{
+					IP:   net.IPv4(0, 0, 0, 0),
+					Mask: net.CIDRMask(0, 0),
+				}
+				ip := net.IPv4(192, 0, 0, 254)
+				route := netlink.Route{LinkIndex: link.Attrs().Index, Dst: dst, Gw: ip}
+				err = netlink.RouteAdd(&route)
+				Expect(err).NotTo(HaveOccurred())
+				return nil
+			})
+			Expect(err).NotTo(HaveOccurred())
+
+			ipvlanAddCheckDelTest(conf, MASTER_NAME, originalNS, targetNS)
+		})
+	}
 })

plugins/main/loopback/loopback.go

@@ -24,7 +24,7 @@ import (
 
 	"github.com/containernetworking/cni/pkg/skel"
 	"github.com/containernetworking/cni/pkg/types"
-	"github.com/containernetworking/cni/pkg/types/current"
+	current "github.com/containernetworking/cni/pkg/types/100"
 	"github.com/containernetworking/cni/pkg/version"
 
 	"github.com/containernetworking/plugins/pkg/ns"
@@ -109,12 +109,19 @@ func cmdAdd(args *skel.CmdArgs) error {
 		// loopback should pass it transparently
 		result = conf.PrevResult
 	} else {
-		loopbackInterface := &current.Interface{Name: args.IfName, Mac: "00:00:00:00:00:00", Sandbox: args.Netns}
-		r := &current.Result{CNIVersion: conf.CNIVersion, Interfaces: []*current.Interface{loopbackInterface}}
+		r := &current.Result{
+			CNIVersion: conf.CNIVersion,
+			Interfaces: []*current.Interface{
+				&current.Interface{
+					Name:    args.IfName,
+					Mac:     "00:00:00:00:00:00",
+					Sandbox: args.Netns,
+				},
+			},
+		}
 
 		if v4Addr != nil {
 			r.IPs = append(r.IPs, &current.IPConfig{
-				Version:   "4",
 				Interface: current.Int(0),
 				Address:   *v4Addr,
 			})
@@ -122,7 +129,6 @@ func cmdAdd(args *skel.CmdArgs) error {
 
 		if v6Addr != nil {
 			r.IPs = append(r.IPs, &current.IPConfig{
-				Version:   "6",
 				Interface: current.Int(0),
 				Address:   *v6Addr,
 			})
@@ -153,7 +159,14 @@ func cmdDel(args *skel.CmdArgs) error {
 		return nil
 	})
 	if err != nil {
-		return err // not tested
+		//  if NetNs is passed down by the Cloud Orchestration Engine, or if it called multiple times
+		// so don't return an error if the device is already removed.
+		// https://github.com/kubernetes/kubernetes/issues/43014#issuecomment-287164444
+		_, ok := err.(ns.NSPathNotExistErr)
+		if ok {
+			return nil
+		}
+		return err
 	}
 
 	return nil

plugins/main/loopback/loopback_test.go

@@ -28,6 +28,10 @@ import (
 	"github.com/onsi/gomega/gexec"
 )
 
+func generateConfig(cniVersion string) *strings.Reader {
+	return strings.NewReader(fmt.Sprintf(`{ "name": "loopback-test", "cniVersion": "%s" }`, cniVersion))
+}
+
 var _ = Describe("Loopback", func() {
 	var (
 		networkNS ns.NetNS
@@ -49,7 +53,6 @@ var _ = Describe("Loopback", func() {
 			fmt.Sprintf("CNI_ARGS=%s", "none"),
 			fmt.Sprintf("CNI_PATH=%s", "/some/test/path"),
 		}
-		command.Stdin = strings.NewReader(`{ "name": "loopback-test", "cniVersion": "0.1.0" }`)
 	})
 
 	AfterEach(func() {
@@ -57,45 +60,53 @@ var _ = Describe("Loopback", func() {
 		Expect(testutils.UnmountNS(networkNS)).To(Succeed())
 	})
 
-	Context("when given a network namespace", func() {
-		It("sets the lo device to UP", func() {
-			command.Env = append(environ, fmt.Sprintf("CNI_COMMAND=%s", "ADD"))
+	for _, ver := range testutils.AllSpecVersions {
+		// Redefine ver inside for scope so real value is picked up by each dynamically defined It()
+		// See Gingkgo's "Patterns for dynamically generating tests" documentation.
+		ver := ver
 
-			session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter)
-			Expect(err).NotTo(HaveOccurred())
+		Context("when given a network namespace", func() {
+			It(fmt.Sprintf("[%s] sets the lo device to UP", ver), func() {
+				command.Stdin = generateConfig(ver)
+				command.Env = append(environ, fmt.Sprintf("CNI_COMMAND=%s", "ADD"))
 
-			Eventually(session).Should(gbytes.Say(`{.*}`))
-			Eventually(session).Should(gexec.Exit(0))
+				session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter)
+				Expect(err).NotTo(HaveOccurred())
 
-			var lo *net.Interface
-			err = networkNS.Do(func(ns.NetNS) error {
-				var err error
-				lo, err = net.InterfaceByName("lo")
-				return err
+				Eventually(session).Should(gbytes.Say(`{.*}`))
+				Eventually(session).Should(gexec.Exit(0))
+
+				var lo *net.Interface
+				err = networkNS.Do(func(ns.NetNS) error {
+					var err error
+					lo, err = net.InterfaceByName("lo")
+					return err
+				})
+				Expect(err).NotTo(HaveOccurred())
+
+				Expect(lo.Flags & net.FlagUp).To(Equal(net.FlagUp))
 			})
-			Expect(err).NotTo(HaveOccurred())
 
-			Expect(lo.Flags & net.FlagUp).To(Equal(net.FlagUp))
-		})
+			It(fmt.Sprintf("[%s] sets the lo device to DOWN", ver), func() {
+				command.Stdin = generateConfig(ver)
+				command.Env = append(environ, fmt.Sprintf("CNI_COMMAND=%s", "DEL"))
 
-		It("sets the lo device to DOWN", func() {
-			command.Env = append(environ, fmt.Sprintf("CNI_COMMAND=%s", "DEL"))
+				session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter)
+				Expect(err).NotTo(HaveOccurred())
 
-			session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter)
-			Expect(err).NotTo(HaveOccurred())
+				Eventually(session).Should(gbytes.Say(``))
+				Eventually(session).Should(gexec.Exit(0))
 
-			Eventually(session).Should(gbytes.Say(``))
-			Eventually(session).Should(gexec.Exit(0))
+				var lo *net.Interface
+				err = networkNS.Do(func(ns.NetNS) error {
+					var err error
+					lo, err = net.InterfaceByName("lo")
+					return err
+				})
+				Expect(err).NotTo(HaveOccurred())
 
-			var lo *net.Interface
-			err = networkNS.Do(func(ns.NetNS) error {
-				var err error
-				lo, err = net.InterfaceByName("lo")
-				return err
+				Expect(lo.Flags & net.FlagUp).NotTo(Equal(net.FlagUp))
 			})
-			Expect(err).NotTo(HaveOccurred())
-
-			Expect(lo.Flags & net.FlagUp).NotTo(Equal(net.FlagUp))
 		})
-	})
+	}
 })

plugins/main/macvlan/README.md

@@ -1,5 +1,5 @@
 
 This document has moved to the [containernetworking/cni.dev](https://github.com/containernetworking/cni.dev) repo.
 
-You can find it online here: https://cni.dev/plugins/main/macvlan/
+You can find it online here: https://cni.dev/plugins/current/main/macvlan/
 

plugins/main/macvlan/macvlan.go

@@ -21,12 +21,11 @@ import (
 	"net"
 	"runtime"
 
-	"github.com/j-keck/arping"
 	"github.com/vishvananda/netlink"
 
 	"github.com/containernetworking/cni/pkg/skel"
 	"github.com/containernetworking/cni/pkg/types"
-	"github.com/containernetworking/cni/pkg/types/current"
+	current "github.com/containernetworking/cni/pkg/types/100"
 	"github.com/containernetworking/cni/pkg/version"
 
 	"github.com/containernetworking/plugins/pkg/ip"
@@ -36,10 +35,6 @@ import (
 	"github.com/containernetworking/plugins/pkg/utils/sysctl"
 )
 
-const (
-	IPv4InterfaceArpProxySysctlTemplate = "net.ipv4.conf.%s.proxy_arp"
-)
-
 type NetConf struct {
 	types.NetConf
 	Master string `json:"master"`
@@ -208,14 +203,6 @@ func createMacvlan(conf *NetConf, ifName string, netns ns.NetNS) (*current.Inter
 	}
 
 	err = netns.Do(func(_ ns.NetNS) error {
-		// TODO: duplicate following lines for ipv6 support, when it will be added in other places
-		ipv4SysctlValueName := fmt.Sprintf(IPv4InterfaceArpProxySysctlTemplate, tmpName)
-		if _, err := sysctl.Sysctl(ipv4SysctlValueName, "1"); err != nil {
-			// remove the newly added link and ignore errors, because we already are in a failed state
-			_ = netlink.LinkDel(mv)
-			return fmt.Errorf("failed to set proxy_arp on newly added interface %q: %v", tmpName, err)
-		}
-
 		err := ip.RenameLink(tmpName, ifName)
 		if err != nil {
 			_ = netlink.LinkDel(mv)
@@ -269,7 +256,10 @@ func cmdAdd(args *skel.CmdArgs) error {
 	}()
 
 	// Assume L2 interface only
-	result := &current.Result{CNIVersion: cniVersion, Interfaces: []*current.Interface{macvlanInterface}}
+	result := &current.Result{
+		CNIVersion: current.ImplementedSpecVersion,
+		Interfaces: []*current.Interface{macvlanInterface},
+	}
 
 	if isLayer3 {
 		// run the IPAM plugin and get back the config to apply
@@ -304,20 +294,11 @@ func cmdAdd(args *skel.CmdArgs) error {
 		}
 
 		err = netns.Do(func(_ ns.NetNS) error {
+			_, _ = sysctl.Sysctl(fmt.Sprintf("net/ipv4/conf/%s/arp_notify", args.IfName), "1")
+
 			if err := ipam.ConfigureIface(args.IfName, result); err != nil {
 				return err
 			}
-
-			contVeth, err := net.InterfaceByName(args.IfName)
-			if err != nil {
-				return fmt.Errorf("failed to look up %q: %v", args.IfName, err)
-			}
-
-			for _, ipc := range result.IPs {
-				if ipc.Version == "4" {
-					_ = arping.GratuitousArpOverIface(ipc.Address.IP, *contVeth)
-				}
-			}
 			return nil
 		})
 		if err != nil {
@@ -377,6 +358,17 @@ func cmdDel(args *skel.CmdArgs) error {
 		return nil
 	})
 
+	if err != nil {
+		//  if NetNs is passed down by the Cloud Orchestration Engine, or if it called multiple times
+		// so don't return an error if the device is already removed.
+		// https://github.com/kubernetes/kubernetes/issues/43014#issuecomment-287164444
+		_, ok := err.(ns.NSPathNotExistErr)
+		if ok {
+			return nil
+		}
+		return err
+	}
+
 	return err
 }
 

plugins/main/ptp/README.md

@@ -1,5 +1,5 @@
 
 This document has moved to the [containernetworking/cni.dev](https://github.com/containernetworking/cni.dev) repo.
 
-You can find it online here: https://cni.dev/plugins/main/ptp/
+You can find it online here: https://cni.dev/plugins/current/main/ptp/
 

plugins/main/ptp/ptp.go

@@ -22,12 +22,11 @@ import (
 	"os"
 	"runtime"
 
-	"github.com/j-keck/arping"
 	"github.com/vishvananda/netlink"
 
 	"github.com/containernetworking/cni/pkg/skel"
 	"github.com/containernetworking/cni/pkg/types"
-	"github.com/containernetworking/cni/pkg/types/current"
+	current "github.com/containernetworking/cni/pkg/types/100"
 	"github.com/containernetworking/cni/pkg/version"
 
 	"github.com/containernetworking/plugins/pkg/ip"
@@ -66,7 +65,7 @@ func setupContainerVeth(netns ns.NetNS, ifName string, mtu int, pr *current.Resu
 	containerInterface := &current.Interface{}
 
 	err := netns.Do(func(hostNS ns.NetNS) error {
-		hostVeth, contVeth0, err := ip.SetupVeth(ifName, mtu, hostNS)
+		hostVeth, contVeth0, err := ip.SetupVeth(ifName, mtu, "", hostNS)
 		if err != nil {
 			return err
 		}
@@ -83,15 +82,15 @@ func setupContainerVeth(netns ns.NetNS, ifName string, mtu int, pr *current.Resu
 
 		pr.Interfaces = []*current.Interface{hostInterface, containerInterface}
 
-		if err = ipam.ConfigureIface(ifName, pr); err != nil {
-			return err
-		}
-
 		contVeth, err := net.InterfaceByName(ifName)
 		if err != nil {
 			return fmt.Errorf("failed to look up %q: %v", ifName, err)
 		}
 
+		if err = ipam.ConfigureIface(ifName, pr); err != nil {
+			return err
+		}
+
 		for _, ipc := range pr.IPs {
 			// Delete the route that was automatically added
 			route := netlink.Route{
@@ -108,7 +107,7 @@ func setupContainerVeth(netns ns.NetNS, ifName string, mtu int, pr *current.Resu
 			}
 
 			addrBits := 32
-			if ipc.Version == "6" {
+			if ipc.Address.IP.To4() == nil {
 				addrBits = 128
 			}
 
@@ -139,13 +138,6 @@ func setupContainerVeth(netns ns.NetNS, ifName string, mtu int, pr *current.Resu
 			}
 		}
 
-		// Send a gratuitous arp for all v4 addresses
-		for _, ipc := range pr.IPs {
-			if ipc.Version == "4" {
-				_ = arping.GratuitousArpOverIface(ipc.Address.IP, *contVeth)
-			}
-		}
-
 		return nil
 	})
 	if err != nil {
@@ -292,6 +284,13 @@ func cmdDel(args *skel.CmdArgs) error {
 	})
 
 	if err != nil {
+		//  if NetNs is passed down by the Cloud Orchestration Engine, or if it called multiple times
+		// so don't return an error if the device is already removed.
+		// https://github.com/kubernetes/kubernetes/issues/43014#issuecomment-287164444
+		_, ok := err.(ns.NSPathNotExistErr)
+		if ok {
+			return nil
+		}
 		return err
 	}
 

plugins/main/ptp/ptp_test.go

@@ -18,10 +18,13 @@ import (
 	"encoding/json"
 	"fmt"
 	"os"
+	"strings"
 
 	"github.com/containernetworking/cni/pkg/skel"
 	"github.com/containernetworking/cni/pkg/types"
-	"github.com/containernetworking/cni/pkg/types/current"
+	types020 "github.com/containernetworking/cni/pkg/types/020"
+	types040 "github.com/containernetworking/cni/pkg/types/040"
+	types100 "github.com/containernetworking/cni/pkg/types/100"
 	"github.com/containernetworking/plugins/pkg/ns"
 	"github.com/containernetworking/plugins/pkg/testutils"
 
@@ -41,7 +44,7 @@ type Net struct {
 	IPAM          *allocator.IPAMConfig  `json:"ipam"`
 	DNS           types.DNS              `json:"dns"`
 	RawPrevResult map[string]interface{} `json:"prevResult,omitempty"`
-	PrevResult    current.Result         `json:"-"`
+	PrevResult    types100.Result        `json:"-"`
 }
 
 func buildOneConfig(netName string, cniVersion string, orig *Net, prevResult types.Result) (*Net, error) {
@@ -87,43 +90,166 @@ func buildOneConfig(netName string, cniVersion string, orig *Net, prevResult typ
 
 }
 
+type tester interface {
+	// verifyResult minimally verifies the Result and returns the interface's IP addresses and MAC address
+	verifyResult(result types.Result, expectedIfName, expectedSandbox string, expectedDNS types.DNS) ([]resultIP, string)
+}
+
+type testerBase struct{}
+
+type testerV10x testerBase
+type testerV04x testerBase
+type testerV03x testerBase
+type testerV01xOr02x testerBase
+
+func newTesterByVersion(version string) tester {
+	switch {
+	case strings.HasPrefix(version, "1.0."):
+		return &testerV10x{}
+	case strings.HasPrefix(version, "0.4."):
+		return &testerV04x{}
+	case strings.HasPrefix(version, "0.3."):
+		return &testerV03x{}
+	default:
+		return &testerV01xOr02x{}
+	}
+}
+
+type resultIP struct {
+	ip string
+	gw string
+}
+
+// verifyResult minimally verifies the Result and returns the interface's IP addresses and MAC address
+func (t *testerV10x) verifyResult(result types.Result, expectedIfName, expectedSandbox string, expectedDNS types.DNS) ([]resultIP, string) {
+	r, err := types100.GetResult(result)
+	Expect(err).NotTo(HaveOccurred())
+
+	Expect(r.Interfaces).To(HaveLen(2))
+	Expect(r.Interfaces[0].Name).To(HavePrefix("veth"))
+	Expect(r.Interfaces[0].Mac).To(HaveLen(17))
+	Expect(r.Interfaces[0].Sandbox).To(BeEmpty())
+	Expect(r.Interfaces[1].Name).To(Equal(expectedIfName))
+	Expect(r.Interfaces[1].Sandbox).To(Equal(expectedSandbox))
+
+	Expect(r.DNS).To(Equal(expectedDNS))
+
+	// Grab IPs from container interface
+	ips := []resultIP{}
+	for _, ipc := range r.IPs {
+		if *ipc.Interface == 1 {
+			ips = append(ips, resultIP{
+				ip: ipc.Address.IP.String(),
+				gw: ipc.Gateway.String(),
+			})
+		}
+	}
+
+	return ips, r.Interfaces[1].Mac
+}
+
+func verify0403(result types.Result, expectedIfName, expectedSandbox string, expectedDNS types.DNS) ([]resultIP, string) {
+	r, err := types040.GetResult(result)
+	Expect(err).NotTo(HaveOccurred())
+
+	Expect(r.Interfaces).To(HaveLen(2))
+	Expect(r.Interfaces[0].Name).To(HavePrefix("veth"))
+	Expect(r.Interfaces[0].Mac).To(HaveLen(17))
+	Expect(r.Interfaces[0].Sandbox).To(BeEmpty())
+	Expect(r.Interfaces[1].Name).To(Equal(expectedIfName))
+	Expect(r.Interfaces[1].Sandbox).To(Equal(expectedSandbox))
+
+	Expect(r.DNS).To(Equal(expectedDNS))
+
+	// Grab IPs from container interface
+	ips := []resultIP{}
+	for _, ipc := range r.IPs {
+		if *ipc.Interface == 1 {
+			ips = append(ips, resultIP{
+				ip: ipc.Address.IP.String(),
+				gw: ipc.Gateway.String(),
+			})
+		}
+	}
+
+	return ips, r.Interfaces[1].Mac
+}
+
+// verifyResult minimally verifies the Result and returns the interface's IP addresses and MAC address
+func (t *testerV04x) verifyResult(result types.Result, expectedIfName, expectedSandbox string, expectedDNS types.DNS) ([]resultIP, string) {
+	return verify0403(result, expectedIfName, expectedSandbox, expectedDNS)
+}
+
+// verifyResult minimally verifies the Result and returns the interface's IP addresses and MAC address
+func (t *testerV03x) verifyResult(result types.Result, expectedIfName, expectedSandbox string, expectedDNS types.DNS) ([]resultIP, string) {
+	return verify0403(result, expectedIfName, expectedSandbox, expectedDNS)
+}
+
+// verifyResult minimally verifies the Result and returns the interface's IP addresses and MAC address
+func (t *testerV01xOr02x) verifyResult(result types.Result, expectedIfName, expectedSandbox string, expectedDNS types.DNS) ([]resultIP, string) {
+	r, err := types020.GetResult(result)
+	Expect(err).NotTo(HaveOccurred())
+
+	ips := []resultIP{}
+	if r.IP4 != nil && r.IP4.IP.IP != nil {
+		ips = append(ips, resultIP{
+			ip: r.IP4.IP.IP.String(),
+			gw: r.IP4.Gateway.String(),
+		})
+	}
+	if r.IP6 != nil && r.IP6.IP.IP != nil {
+		ips = append(ips, resultIP{
+			ip: r.IP6.IP.IP.String(),
+			gw: r.IP6.Gateway.String(),
+		})
+	}
+
+	// 0.2 and earlier don't return MAC address
+	return ips, ""
+}
+
 var _ = Describe("ptp Operations", func() {
-	var originalNS ns.NetNS
+	var originalNS, targetNS ns.NetNS
+	var dataDir string
 
 	BeforeEach(func() {
 		// Create a new NetNS so we don't modify the host
 		var err error
 		originalNS, err = testutils.NewNS()
 		Expect(err).NotTo(HaveOccurred())
+		targetNS, err = testutils.NewNS()
+		Expect(err).NotTo(HaveOccurred())
+
+		dataDir, err = os.MkdirTemp("", "ptp_test")
+		Expect(err).NotTo(HaveOccurred())
 	})
 
 	AfterEach(func() {
+		Expect(os.RemoveAll(dataDir)).To(Succeed())
 		Expect(originalNS.Close()).To(Succeed())
 		Expect(testutils.UnmountNS(originalNS)).To(Succeed())
+		Expect(targetNS.Close()).To(Succeed())
+		Expect(testutils.UnmountNS(targetNS)).To(Succeed())
 	})
 
-	doTest := func(conf string, numIPs int, expectedDNSConf types.DNS) {
+	doTest := func(conf, cniVersion string, numIPs int, expectedDNSConf types.DNS, targetNS ns.NetNS) {
 		const IFNAME = "ptp0"
 
-		targetNs, err := testutils.NewNS()
-		Expect(err).NotTo(HaveOccurred())
-		defer targetNs.Close()
-
 		args := &skel.CmdArgs{
 			ContainerID: "dummy",
-			Netns:       targetNs.Path(),
+			Netns:       targetNS.Path(),
 			IfName:      IFNAME,
 			StdinData:   []byte(conf),
 		}
 
-		var resI types.Result
-		var res *current.Result
+		var result types.Result
 
 		// Execute the plugin with the ADD command, creating the veth endpoints
-		err = originalNS.Do(func(ns.NetNS) error {
+		err := originalNS.Do(func(ns.NetNS) error {
 			defer GinkgoRecover()
 
-			resI, _, err = testutils.CmdAddWithArgs(args, func() error {
+			var err error
+			result, _, err = testutils.CmdAddWithArgs(args, func() error {
 				return cmdAdd(args)
 			})
 			Expect(err).NotTo(HaveOccurred())
@@ -131,32 +257,25 @@ var _ = Describe("ptp Operations", func() {
 		})
 		Expect(err).NotTo(HaveOccurred())
 
-		res, err = current.NewResultFromResult(resI)
-		Expect(err).NotTo(HaveOccurred())
+		t := newTesterByVersion(cniVersion)
+		ips, mac := t.verifyResult(result, IFNAME, targetNS.Path(), expectedDNSConf)
+		Expect(len(ips)).To(Equal(numIPs))
 
 		// Make sure ptp link exists in the target namespace
 		// Then, ping the gateway
-		seenIPs := 0
-
-		wantMac := ""
-		err = targetNs.Do(func(ns.NetNS) error {
+		err = targetNS.Do(func(ns.NetNS) error {
 			defer GinkgoRecover()
 
 			link, err := netlink.LinkByName(IFNAME)
 			Expect(err).NotTo(HaveOccurred())
-			wantMac = link.Attrs().HardwareAddr.String()
+			if mac != "" {
+				Expect(mac).To(Equal(link.Attrs().HardwareAddr.String()))
+			}
 
-			for _, ipc := range res.IPs {
-				if *ipc.Interface != 1 {
-					continue
-				}
-				seenIPs += 1
-				saddr := ipc.Address.IP.String()
-				daddr := ipc.Gateway.String()
-				fmt.Fprintln(GinkgoWriter, "ping", saddr, "->", daddr)
-
-				if err := testutils.Ping(saddr, daddr, (ipc.Version == "6"), 30); err != nil {
-					return fmt.Errorf("ping %s -> %s failed: %s", saddr, daddr, err)
+			for _, ipc := range ips {
+				fmt.Fprintln(GinkgoWriter, "ping", ipc.ip, "->", ipc.gw)
+				if err := testutils.Ping(ipc.ip, ipc.gw, 30); err != nil {
+					return fmt.Errorf("ping %s -> %s failed: %s", ipc.ip, ipc.gw, err)
 				}
 			}
 
@@ -164,121 +283,6 @@ var _ = Describe("ptp Operations", func() {
 		})
 		Expect(err).NotTo(HaveOccurred())
 
-		Expect(seenIPs).To(Equal(numIPs))
-
-		// make sure the interfaces are correct
-		Expect(res.Interfaces).To(HaveLen(2))
-
-		Expect(res.Interfaces[0].Name).To(HavePrefix("veth"))
-		Expect(res.Interfaces[0].Mac).To(HaveLen(17))
-		Expect(res.Interfaces[0].Sandbox).To(BeEmpty())
-
-		Expect(res.Interfaces[1].Name).To(Equal(IFNAME))
-		Expect(res.Interfaces[1].Mac).To(Equal(wantMac))
-		Expect(res.Interfaces[1].Sandbox).To(Equal(targetNs.Path()))
-
-		// make sure DNS is correct
-		Expect(res.DNS).To(Equal(expectedDNSConf))
-
-		// Call the plugins with the DEL command, deleting the veth endpoints
-		err = originalNS.Do(func(ns.NetNS) error {
-			defer GinkgoRecover()
-
-			err := testutils.CmdDelWithArgs(args, func() error {
-				return cmdDel(args)
-			})
-			Expect(err).NotTo(HaveOccurred())
-			return nil
-		})
-		Expect(err).NotTo(HaveOccurred())
-
-		// Make sure ptp link has been deleted
-		err = targetNs.Do(func(ns.NetNS) error {
-			defer GinkgoRecover()
-
-			link, err := netlink.LinkByName(IFNAME)
-			Expect(err).To(HaveOccurred())
-			Expect(link).To(BeNil())
-			return nil
-		})
-		Expect(err).NotTo(HaveOccurred())
-	}
-
-	doTestv4 := func(conf string, netName string, numIPs int) {
-		const IFNAME = "ptp0"
-
-		targetNs, err := testutils.NewNS()
-		Expect(err).NotTo(HaveOccurred())
-		defer targetNs.Close()
-
-		args := &skel.CmdArgs{
-			ContainerID: "dummy",
-			Netns:       targetNs.Path(),
-			IfName:      IFNAME,
-			StdinData:   []byte(conf),
-		}
-
-		var resI types.Result
-		var res *current.Result
-
-		// Execute the plugin with the ADD command, creating the veth endpoints
-		err = originalNS.Do(func(ns.NetNS) error {
-			defer GinkgoRecover()
-
-			resI, _, err = testutils.CmdAddWithArgs(args, func() error {
-				return cmdAdd(args)
-			})
-			Expect(err).NotTo(HaveOccurred())
-			return nil
-		})
-		Expect(err).NotTo(HaveOccurred())
-
-		res, err = current.NewResultFromResult(resI)
-		Expect(err).NotTo(HaveOccurred())
-
-		// Make sure ptp link exists in the target namespace
-		// Then, ping the gateway
-		seenIPs := 0
-
-		wantMac := ""
-		err = targetNs.Do(func(ns.NetNS) error {
-			defer GinkgoRecover()
-
-			link, err := netlink.LinkByName(IFNAME)
-			Expect(err).NotTo(HaveOccurred())
-			wantMac = link.Attrs().HardwareAddr.String()
-
-			for _, ipc := range res.IPs {
-				if *ipc.Interface != 1 {
-					continue
-				}
-				seenIPs += 1
-				saddr := ipc.Address.IP.String()
-				daddr := ipc.Gateway.String()
-				fmt.Fprintln(GinkgoWriter, "ping", saddr, "->", daddr)
-
-				if err := testutils.Ping(saddr, daddr, (ipc.Version == "6"), 30); err != nil {
-					return fmt.Errorf("ping %s -> %s failed: %s", saddr, daddr, err)
-				}
-			}
-
-			return nil
-		})
-		Expect(err).NotTo(HaveOccurred())
-
-		Expect(seenIPs).To(Equal(numIPs))
-
-		// make sure the interfaces are correct
-		Expect(res.Interfaces).To(HaveLen(2))
-
-		Expect(res.Interfaces[0].Name).To(HavePrefix("veth"))
-		Expect(res.Interfaces[0].Mac).To(HaveLen(17))
-		Expect(res.Interfaces[0].Sandbox).To(BeEmpty())
-
-		Expect(res.Interfaces[1].Name).To(Equal(IFNAME))
-		Expect(res.Interfaces[1].Mac).To(Equal(wantMac))
-		Expect(res.Interfaces[1].Sandbox).To(Equal(targetNs.Path()))
-
 		// call CmdCheck
 		n := &Net{}
 		err = json.Unmarshal([]byte(conf), &n)
@@ -287,8 +291,7 @@ var _ = Describe("ptp Operations", func() {
 		n.IPAM, _, err = allocator.LoadIPAMConfig([]byte(conf), "")
 		Expect(err).NotTo(HaveOccurred())
 
-		cniVersion := "0.4.0"
-		newConf, err := buildOneConfig(netName, cniVersion, n, res)
+		newConf, err := buildOneConfig(n.Name, cniVersion, n, result)
 		Expect(err).NotTo(HaveOccurred())
 
 		confString, err := json.Marshal(newConf)
@@ -299,11 +302,13 @@ var _ = Describe("ptp Operations", func() {
 		// CNI Check host-device in the target namespace
 		err = originalNS.Do(func(ns.NetNS) error {
 			defer GinkgoRecover()
-			var err error
-			err = testutils.CmdCheckWithArgs(args, func() error { return cmdCheck(args) })
-			return err
+			return testutils.CmdCheckWithArgs(args, func() error { return cmdCheck(args) })
 		})
-		Expect(err).NotTo(HaveOccurred())
+		if testutils.SpecVersionHasCHECK(cniVersion) {
+			Expect(err).NotTo(HaveOccurred())
+		} else {
+			Expect(err).To(MatchError("config version does not allow CHECK"))
+		}
 
 		args.StdinData = []byte(conf)
 
@@ -320,7 +325,7 @@ var _ = Describe("ptp Operations", func() {
 		Expect(err).NotTo(HaveOccurred())
 
 		// Make sure ptp link has been deleted
-		err = targetNs.Do(func(ns.NetNS) error {
+		err = targetNS.Do(func(ns.NetNS) error {
 			defer GinkgoRecover()
 
 			link, err := netlink.LinkByName(IFNAME)
@@ -331,227 +336,200 @@ var _ = Describe("ptp Operations", func() {
 		Expect(err).NotTo(HaveOccurred())
 	}
 
-	It("configures and deconfigures a ptp link with ADD/DEL", func() {
-		dnsConf := types.DNS{
-			Nameservers: []string{"10.1.2.123"},
-			Domain:      "some.domain.test",
-			Search:      []string{"search.test"},
-			Options:     []string{"option1:foo"},
-		}
-		dnsConfBytes, err := json.Marshal(dnsConf)
-		Expect(err).NotTo(HaveOccurred())
+	for _, ver := range testutils.AllSpecVersions {
+		// Redefine ver inside for scope so real value is picked up by each dynamically defined It()
+		// See Gingkgo's "Patterns for dynamically generating tests" documentation.
+		ver := ver
 
-		conf := fmt.Sprintf(`{
-    "cniVersion": "0.3.1",
-    "name": "mynet",
-    "type": "ptp",
-    "ipMasq": true,
-    "mtu": 5000,
-    "ipam": {
-        "type": "host-local",
-        "subnet": "10.1.2.0/24"
-    },
-    "dns": %s
-}`, string(dnsConfBytes))
-
-		doTest(conf, 1, dnsConf)
-	})
-
-	It("configures and deconfigures a dual-stack ptp link with ADD/DEL", func() {
-		conf := `{
-    "cniVersion": "0.3.1",
-    "name": "mynet",
-    "type": "ptp",
-    "ipMasq": true,
-    "mtu": 5000,
-    "ipam": {
-        "type": "host-local",
-		"ranges": [
-			[{ "subnet": "10.1.2.0/24"}],
-			[{ "subnet": "2001:db8:1::0/66"}]
-		]
-    }
-}`
-
-		doTest(conf, 2, types.DNS{})
-	})
-
-	It("does not override IPAM DNS settings if no DNS settings provided", func() {
-		ipamDNSConf := types.DNS{
-			Nameservers: []string{"10.1.2.123"},
-			Domain:      "some.domain.test",
-			Search:      []string{"search.test"},
-			Options:     []string{"option1:foo"},
-		}
-		resolvConfPath, err := testutils.TmpResolvConf(ipamDNSConf)
-		Expect(err).NotTo(HaveOccurred())
-		defer os.RemoveAll(resolvConfPath)
-
-		conf := fmt.Sprintf(`{
-    "cniVersion": "0.3.1",
-    "name": "mynet",
-    "type": "ptp",
-    "ipMasq": true,
-    "mtu": 5000,
-    "ipam": {
-        "type": "host-local",
-        "subnet": "10.1.2.0/24",
-        "resolvConf": "%s"
-    }
-}`, resolvConfPath)
-
-		doTest(conf, 1, ipamDNSConf)
-	})
-
-	It("overrides IPAM DNS settings if any DNS settings provided", func() {
-		ipamDNSConf := types.DNS{
-			Nameservers: []string{"10.1.2.123"},
-			Domain:      "some.domain.test",
-			Search:      []string{"search.test"},
-			Options:     []string{"option1:foo"},
-		}
-		resolvConfPath, err := testutils.TmpResolvConf(ipamDNSConf)
-		Expect(err).NotTo(HaveOccurred())
-		defer os.RemoveAll(resolvConfPath)
-
-		for _, ptpDNSConf := range []types.DNS{
-			{
-				Nameservers: []string{"10.1.2.234"},
-			},
-			{
-				Domain: "someother.domain.test",
-			},
-			{
-				Search: []string{"search.elsewhere.test"},
-			},
-			{
-				Options: []string{"option2:bar"},
-			},
-		} {
-			dnsConfBytes, err := json.Marshal(ptpDNSConf)
+		It(fmt.Sprintf("[%s] configures and deconfigures a ptp link with ADD/DEL", ver), func() {
+			dnsConf := types.DNS{
+				Nameservers: []string{"10.1.2.123"},
+				Domain:      "some.domain.test",
+				Search:      []string{"search.test"},
+				Options:     []string{"option1:foo"},
+			}
+			dnsConfBytes, err := json.Marshal(dnsConf)
 			Expect(err).NotTo(HaveOccurred())
 
 			conf := fmt.Sprintf(`{
-    "cniVersion": "0.3.1",
-    "name": "mynet",
-    "type": "ptp",
-    "ipMasq": true,
-    "mtu": 5000,
-    "ipam": {
-        "type": "host-local",
-        "subnet": "10.1.2.0/24",
-        "resolvConf": "%s"
-    },
-    "dns": %s
-}`, resolvConfPath, string(dnsConfBytes))
+			    "cniVersion": "%s",
+			    "name": "mynet",
+			    "type": "ptp",
+			    "ipMasq": true,
+			    "mtu": 5000,
+			    "ipam": {
+				"type": "host-local",
+				"subnet": "10.1.2.0/24",
+				"dataDir": "%s"
+			    },
+			    "dns": %s
+			}`, ver, dataDir, string(dnsConfBytes))
 
-			doTest(conf, 1, ptpDNSConf)
-		}
-	})
+			doTest(conf, ver, 1, dnsConf, targetNS)
+		})
 
-	It("overrides IPAM DNS settings if any empty list DNS settings provided", func() {
-		ipamDNSConf := types.DNS{
-			Nameservers: []string{"10.1.2.123"},
-			Domain:      "some.domain.test",
-			Search:      []string{"search.test"},
-			Options:     []string{"option1:foo"},
-		}
-		resolvConfPath, err := testutils.TmpResolvConf(ipamDNSConf)
-		Expect(err).NotTo(HaveOccurred())
-		defer os.RemoveAll(resolvConfPath)
+		It(fmt.Sprintf("[%s] configures and deconfigures a dual-stack ptp link with ADD/DEL", ver), func() {
+			conf := fmt.Sprintf(`{
+			    "cniVersion": "%s",
+			    "name": "mynet",
+			    "type": "ptp",
+			    "ipMasq": true,
+			    "mtu": 5000,
+			    "ipam": {
+				"type": "host-local",
+				"ranges": [
+					[{ "subnet": "10.1.2.0/24"}],
+					[{ "subnet": "2001:db8:1::0/66"}]
+				],
+				"dataDir": "%s"
+			    }
+			}`, ver, dataDir)
 
-		conf := fmt.Sprintf(`{
-    "cniVersion": "0.3.1",
-    "name": "mynet",
-    "type": "ptp",
-    "ipMasq": true,
-    "mtu": 5000,
-    "ipam": {
-        "type": "host-local",
-        "subnet": "10.1.2.0/24",
-        "resolvConf": "%s"
-    },
-    "dns": {
-        "nameservers": [],
-        "search": [],
-        "options": []
-    }
-}`, resolvConfPath)
+			doTest(conf, ver, 2, types.DNS{}, targetNS)
+		})
 
-		doTest(conf, 1, types.DNS{})
-	})
+		It(fmt.Sprintf("[%s] does not override IPAM DNS settings if no DNS settings provided", ver), func() {
+			ipamDNSConf := types.DNS{
+				Nameservers: []string{"10.1.2.123"},
+				Domain:      "some.domain.test",
+				Search:      []string{"search.test"},
+				Options:     []string{"option1:foo"},
+			}
+			resolvConfPath, err := testutils.TmpResolvConf(ipamDNSConf)
+			Expect(err).NotTo(HaveOccurred())
+			defer os.RemoveAll(resolvConfPath)
 
-	It("deconfigures an unconfigured ptp link with DEL", func() {
-		const IFNAME = "ptp0"
+			conf := fmt.Sprintf(`{
+			    "cniVersion": "%s",
+			    "name": "mynet",
+			    "type": "ptp",
+			    "ipMasq": true,
+			    "mtu": 5000,
+			    "ipam": {
+				"type": "host-local",
+				"subnet": "10.1.2.0/24",
+				"resolvConf": "%s",
+				"dataDir": "%s"
+			    }
+			}`, ver, resolvConfPath, dataDir)
 
-		conf := `{
-    "cniVersion": "0.3.0",
-    "name": "mynet",
-    "type": "ptp",
-    "ipMasq": true,
-    "mtu": 5000,
-    "ipam": {
-        "type": "host-local",
-        "subnet": "10.1.2.0/24"
-    }
-}`
+			doTest(conf, ver, 1, ipamDNSConf, targetNS)
+		})
 
-		targetNs, err := testutils.NewNS()
-		Expect(err).NotTo(HaveOccurred())
-		defer targetNs.Close()
+		It(fmt.Sprintf("[%s] overrides IPAM DNS settings if any DNS settings provided", ver), func() {
+			ipamDNSConf := types.DNS{
+				Nameservers: []string{"10.1.2.123"},
+				Domain:      "some.domain.test",
+				Search:      []string{"search.test"},
+				Options:     []string{"option1:foo"},
+			}
+			resolvConfPath, err := testutils.TmpResolvConf(ipamDNSConf)
+			Expect(err).NotTo(HaveOccurred())
+			defer os.RemoveAll(resolvConfPath)
 
-		args := &skel.CmdArgs{
-			ContainerID: "dummy",
-			Netns:       targetNs.Path(),
-			IfName:      IFNAME,
-			StdinData:   []byte(conf),
-		}
+			for _, ptpDNSConf := range []types.DNS{
+				{
+					Nameservers: []string{"10.1.2.234"},
+				},
+				{
+					Domain: "someother.domain.test",
+				},
+				{
+					Search: []string{"search.elsewhere.test"},
+				},
+				{
+					Options: []string{"option2:bar"},
+				},
+			} {
+				dnsConfBytes, err := json.Marshal(ptpDNSConf)
+				Expect(err).NotTo(HaveOccurred())
 
-		// Call the plugins with the DEL command. It should not error even though the veth doesn't exist.
-		err = originalNS.Do(func(ns.NetNS) error {
-			defer GinkgoRecover()
+				conf := fmt.Sprintf(`{
+				    "cniVersion": "%s",
+				    "name": "mynet",
+				    "type": "ptp",
+				    "ipMasq": true,
+				    "mtu": 5000,
+				    "ipam": {
+					"type": "host-local",
+					"subnet": "10.1.2.0/24",
+					"resolvConf": "%s",
+					"dataDir": "%s"
+				    },
+				    "dns": %s
+				}`, ver, resolvConfPath, dataDir, string(dnsConfBytes))
 
-			err := testutils.CmdDelWithArgs(args, func() error {
-				return cmdDel(args)
+				doTest(conf, ver, 1, ptpDNSConf, targetNS)
+			}
+		})
+
+		It(fmt.Sprintf("[%s] overrides IPAM DNS settings if any empty list DNS settings provided", ver), func() {
+			ipamDNSConf := types.DNS{
+				Nameservers: []string{"10.1.2.123"},
+				Domain:      "some.domain.test",
+				Search:      []string{"search.test"},
+				Options:     []string{"option1:foo"},
+			}
+			resolvConfPath, err := testutils.TmpResolvConf(ipamDNSConf)
+			Expect(err).NotTo(HaveOccurred())
+			defer os.RemoveAll(resolvConfPath)
+
+			conf := fmt.Sprintf(`{
+			    "cniVersion": "%s",
+			    "name": "mynet",
+			    "type": "ptp",
+			    "ipMasq": true,
+			    "mtu": 5000,
+			    "ipam": {
+				"type": "host-local",
+				"subnet": "10.1.2.0/24",
+				"dataDir": "%s",
+				"resolvConf": "%s"
+			    },
+			    "dns": {
+				"nameservers": [],
+				"search": [],
+				"options": []
+			    }
+			}`, ver, dataDir, resolvConfPath)
+
+			doTest(conf, ver, 1, types.DNS{}, targetNS)
+		})
+
+		It(fmt.Sprintf("[%s] deconfigures an unconfigured ptp link with DEL", ver), func() {
+			const IFNAME = "ptp0"
+
+			conf := fmt.Sprintf(`{
+			    "cniVersion": "%s",
+			    "name": "mynet",
+			    "type": "ptp",
+			    "ipMasq": true,
+			    "mtu": 5000,
+			    "ipam": {
+				"type": "host-local",
+				"dataDir": "%s",
+				"subnet": "10.1.2.0/24"
+			    }
+			}`, ver, dataDir)
+
+			args := &skel.CmdArgs{
+				ContainerID: "dummy",
+				Netns:       targetNS.Path(),
+				IfName:      IFNAME,
+				StdinData:   []byte(conf),
+			}
+
+			// Call the plugins with the DEL command. It should not error even though the veth doesn't exist.
+			err := originalNS.Do(func(ns.NetNS) error {
+				defer GinkgoRecover()
+
+				err := testutils.CmdDelWithArgs(args, func() error {
+					return cmdDel(args)
+				})
+				Expect(err).NotTo(HaveOccurred())
+				return nil
 			})
 			Expect(err).NotTo(HaveOccurred())
-			return nil
 		})
-		Expect(err).NotTo(HaveOccurred())
-	})
-
-	It("configures and deconfigures a CNI V4 ptp link with ADD/DEL", func() {
-		conf := `{
-    "cniVersion": "0.4.0",
-    "name": "ptpNetv4",
-    "type": "ptp",
-    "ipMasq": true,
-    "mtu": 5000,
-    "ipam": {
-        "type": "host-local",
-        "subnet": "10.1.2.0/24"
-    }
-}`
-
-		doTestv4(conf, "ptpNetv4", 1)
-	})
-
-	It("configures and deconfigures a CNI V4 dual-stack ptp link with ADD/DEL", func() {
-		conf := `{
-    "cniVersion": "0.4.0",
-    "name": "ptpNetv4ds",
-    "type": "ptp",
-    "ipMasq": true,
-    "mtu": 5000,
-    "ipam": {
-        "type": "host-local",
-		"ranges": [
-			[{ "subnet": "10.1.2.0/24"}],
-			[{ "subnet": "2001:db8:1::0/66"}]
-		]
-    }
-}`
-
-		doTestv4(conf, "ptpNetv4ds", 2)
-	})
+	}
 })

plugins/main/vlan/vlan.go

@@ -24,7 +24,7 @@ import (
 
 	"github.com/containernetworking/cni/pkg/skel"
 	"github.com/containernetworking/cni/pkg/types"
-	"github.com/containernetworking/cni/pkg/types/current"
+	current "github.com/containernetworking/cni/pkg/types/100"
 	"github.com/containernetworking/cni/pkg/version"
 
 	"github.com/containernetworking/plugins/pkg/ip"
@@ -213,6 +213,17 @@ func cmdDel(args *skel.CmdArgs) error {
 		return err
 	})
 
+	if err != nil {
+		//  if NetNs is passed down by the Cloud Orchestration Engine, or if it called multiple times
+		// so don't return an error if the device is already removed.
+		// https://github.com/kubernetes/kubernetes/issues/43014#issuecomment-287164444
+		_, ok := err.(ns.NSPathNotExistErr)
+		if ok {
+			return nil
+		}
+		return err
+	}
+
 	return err
 }
 

plugins/main/vlan/vlan_test.go

@@ -18,11 +18,15 @@ import (
 	"encoding/json"
 	"fmt"
 	"net"
+	"os"
+	"strings"
 	"syscall"
 
 	"github.com/containernetworking/cni/pkg/skel"
 	"github.com/containernetworking/cni/pkg/types"
-	"github.com/containernetworking/cni/pkg/types/current"
+	types020 "github.com/containernetworking/cni/pkg/types/020"
+	types040 "github.com/containernetworking/cni/pkg/types/040"
+	types100 "github.com/containernetworking/cni/pkg/types/100"
 	"github.com/containernetworking/plugins/pkg/ns"
 	"github.com/containernetworking/plugins/pkg/testutils"
 
@@ -45,7 +49,7 @@ type Net struct {
 	IPAM          *allocator.IPAMConfig  `json:"ipam"`
 	DNS           types.DNS              `json:"dns"`
 	RawPrevResult map[string]interface{} `json:"prevResult,omitempty"`
-	PrevResult    current.Result         `json:"-"`
+	PrevResult    types100.Result        `json:"-"`
 }
 
 func buildOneConfig(netName string, cniVersion string, orig *Net, prevResult types.Result) (*Net, error) {
@@ -91,14 +95,90 @@ func buildOneConfig(netName string, cniVersion string, orig *Net, prevResult typ
 
 }
 
+type tester interface {
+	// verifyResult minimally verifies the Result and returns the interface's MAC address
+	verifyResult(result types.Result, name string) string
+}
+
+type testerBase struct{}
+
+type testerV10x testerBase
+type testerV04x testerBase
+type testerV03x testerBase
+type testerV01xOr02x testerBase
+
+func newTesterByVersion(version string) tester {
+	switch {
+	case strings.HasPrefix(version, "1.0."):
+		return &testerV10x{}
+	case strings.HasPrefix(version, "0.4."):
+		return &testerV04x{}
+	case strings.HasPrefix(version, "0.3."):
+		return &testerV03x{}
+	default:
+		return &testerV01xOr02x{}
+	}
+}
+
+// verifyResult minimally verifies the Result and returns the interface's MAC address
+func (t *testerV10x) verifyResult(result types.Result, name string) string {
+	r, err := types100.GetResult(result)
+	Expect(err).NotTo(HaveOccurred())
+
+	Expect(len(r.Interfaces)).To(Equal(1))
+	Expect(r.Interfaces[0].Name).To(Equal(name))
+	Expect(len(r.IPs)).To(Equal(1))
+
+	return r.Interfaces[0].Mac
+}
+
+func verify0403(result types.Result, name string) string {
+	r, err := types040.GetResult(result)
+	Expect(err).NotTo(HaveOccurred())
+
+	Expect(len(r.Interfaces)).To(Equal(1))
+	Expect(r.Interfaces[0].Name).To(Equal(name))
+	Expect(len(r.IPs)).To(Equal(1))
+
+	return r.Interfaces[0].Mac
+}
+
+// verifyResult minimally verifies the Result and returns the interface's MAC address
+func (t *testerV04x) verifyResult(result types.Result, name string) string {
+	return verify0403(result, name)
+}
+
+// verifyResult minimally verifies the Result and returns the interface's MAC address
+func (t *testerV03x) verifyResult(result types.Result, name string) string {
+	return verify0403(result, name)
+}
+
+// verifyResult minimally verifies the Result and returns the interface's MAC address
+func (t *testerV01xOr02x) verifyResult(result types.Result, name string) string {
+	r, err := types020.GetResult(result)
+	Expect(err).NotTo(HaveOccurred())
+
+	Expect(r.IP4.IP.IP).NotTo(BeNil())
+	Expect(r.IP6).To(BeNil())
+
+	// 0.2 and earlier don't return MAC address
+	return ""
+}
+
 var _ = Describe("vlan Operations", func() {
-	var originalNS ns.NetNS
+	var originalNS, targetNS ns.NetNS
+	var dataDir string
 
 	BeforeEach(func() {
 		// Create a new NetNS so we don't modify the host
 		var err error
 		originalNS, err = testutils.NewNS()
 		Expect(err).NotTo(HaveOccurred())
+		targetNS, err = testutils.NewNS()
+		Expect(err).NotTo(HaveOccurred())
+
+		dataDir, err = os.MkdirTemp("", "vlan_test")
+		Expect(err).NotTo(HaveOccurred())
 
 		err = originalNS.Do(func(ns.NetNS) error {
 			defer GinkgoRecover()
@@ -120,376 +200,290 @@ var _ = Describe("vlan Operations", func() {
 	})
 
 	AfterEach(func() {
+		Expect(os.RemoveAll(dataDir)).To(Succeed())
 		Expect(originalNS.Close()).To(Succeed())
 		Expect(testutils.UnmountNS(originalNS)).To(Succeed())
+		Expect(targetNS.Close()).To(Succeed())
+		Expect(testutils.UnmountNS(targetNS)).To(Succeed())
 	})
 
-	It("creates an vlan link in a non-default namespace with given MTU", func() {
-		conf := &NetConf{
-			NetConf: types.NetConf{
-				CNIVersion: "0.3.0",
-				Name:       "testConfig",
-				Type:       "vlan",
-			},
-			Master: MASTER_NAME,
-			VlanId: 33,
-			MTU:    1500,
-		}
+	for _, ver := range testutils.AllSpecVersions {
+		// Redefine ver inside for scope so real value is picked up by each dynamically defined It()
+		// See Gingkgo's "Patterns for dynamically generating tests" documentation.
+		ver := ver
 
-		// Create vlan in other namespace
-		targetNs, err := testutils.NewNS()
-		Expect(err).NotTo(HaveOccurred())
-		defer targetNs.Close()
+		It(fmt.Sprintf("[%s] creates an vlan link in a non-default namespace with given MTU", ver), func() {
+			conf := &NetConf{
+				NetConf: types.NetConf{
+					CNIVersion: ver,
+					Name:       "testConfig",
+					Type:       "vlan",
+				},
+				Master: MASTER_NAME,
+				VlanId: 33,
+				MTU:    1500,
+			}
 
-		err = originalNS.Do(func(ns.NetNS) error {
-			defer GinkgoRecover()
+			// Create vlan in other namespace
+			err := originalNS.Do(func(ns.NetNS) error {
+				defer GinkgoRecover()
 
-			_, err := createVlan(conf, "foobar0", targetNs)
-			Expect(err).NotTo(HaveOccurred())
-			return nil
-		})
-
-		Expect(err).NotTo(HaveOccurred())
-
-		// Make sure vlan link exists in the target namespace
-		err = targetNs.Do(func(ns.NetNS) error {
-			defer GinkgoRecover()
-
-			link, err := netlink.LinkByName("foobar0")
-			Expect(err).NotTo(HaveOccurred())
-			Expect(link.Attrs().Name).To(Equal("foobar0"))
-			Expect(link.Attrs().MTU).To(Equal(1500))
-			return nil
-		})
-		Expect(err).NotTo(HaveOccurred())
-	})
-
-	It("creates an vlan link in a non-default namespace with master's MTU", func() {
-		conf := &NetConf{
-			NetConf: types.NetConf{
-				CNIVersion: "0.3.0",
-				Name:       "testConfig",
-				Type:       "vlan",
-			},
-			Master: MASTER_NAME,
-			VlanId: 33,
-		}
-
-		// Create vlan in other namespace
-		targetNs, err := testutils.NewNS()
-		Expect(err).NotTo(HaveOccurred())
-		defer targetNs.Close()
-
-		err = originalNS.Do(func(ns.NetNS) error {
-			defer GinkgoRecover()
-
-			m, err := netlink.LinkByName(MASTER_NAME)
-			Expect(err).NotTo(HaveOccurred())
-			err = netlink.LinkSetMTU(m, 1200)
-			Expect(err).NotTo(HaveOccurred())
-
-			_, err = createVlan(conf, "foobar0", targetNs)
-			Expect(err).NotTo(HaveOccurred())
-			return nil
-		})
-
-		Expect(err).NotTo(HaveOccurred())
-
-		// Make sure vlan link exists in the target namespace
-		err = targetNs.Do(func(ns.NetNS) error {
-			defer GinkgoRecover()
-
-			link, err := netlink.LinkByName("foobar0")
-			Expect(err).NotTo(HaveOccurred())
-			Expect(link.Attrs().Name).To(Equal("foobar0"))
-			Expect(link.Attrs().MTU).To(Equal(1200))
-			return nil
-		})
-		Expect(err).NotTo(HaveOccurred())
-	})
-
-	It("configures and deconfigures an vlan link with ADD/DEL", func() {
-		const IFNAME = "eth0"
-
-		conf := fmt.Sprintf(`{
-    "cniVersion": "0.3.0",
-    "name": "mynet",
-    "type": "vlan",
-    "master": "%s",
-    "ipam": {
-        "type": "host-local",
-        "subnet": "10.1.2.0/24"
-    }
-}`, MASTER_NAME)
-
-		targetNs, err := testutils.NewNS()
-		Expect(err).NotTo(HaveOccurred())
-		defer targetNs.Close()
-
-		args := &skel.CmdArgs{
-			ContainerID: "dummy",
-			Netns:       targetNs.Path(),
-			IfName:      IFNAME,
-			StdinData:   []byte(conf),
-		}
-
-		var result *current.Result
-		err = originalNS.Do(func(ns.NetNS) error {
-			defer GinkgoRecover()
-
-			r, _, err := testutils.CmdAddWithArgs(args, func() error {
-				return cmdAdd(args)
+				_, err := createVlan(conf, "foobar0", targetNS)
+				Expect(err).NotTo(HaveOccurred())
+				return nil
 			})
 			Expect(err).NotTo(HaveOccurred())
 
-			result, err = current.GetResult(r)
-			Expect(err).NotTo(HaveOccurred())
+			// Make sure vlan link exists in the target namespace
+			err = targetNS.Do(func(ns.NetNS) error {
+				defer GinkgoRecover()
 
-			Expect(len(result.Interfaces)).To(Equal(1))
-			Expect(result.Interfaces[0].Name).To(Equal(IFNAME))
-			Expect(len(result.IPs)).To(Equal(1))
-			return nil
-		})
-		Expect(err).NotTo(HaveOccurred())
-
-		// Make sure vlan link exists in the target namespace
-		err = targetNs.Do(func(ns.NetNS) error {
-			defer GinkgoRecover()
-
-			link, err := netlink.LinkByName(IFNAME)
-			Expect(err).NotTo(HaveOccurred())
-			Expect(link.Attrs().Name).To(Equal(IFNAME))
-
-			hwaddr, err := net.ParseMAC(result.Interfaces[0].Mac)
-			Expect(err).NotTo(HaveOccurred())
-			Expect(link.Attrs().HardwareAddr).To(Equal(hwaddr))
-
-			addrs, err := netlink.AddrList(link, syscall.AF_INET)
-			Expect(err).NotTo(HaveOccurred())
-			Expect(len(addrs)).To(Equal(1))
-			return nil
-		})
-		Expect(err).NotTo(HaveOccurred())
-
-		err = originalNS.Do(func(ns.NetNS) error {
-			defer GinkgoRecover()
-
-			err = testutils.CmdDelWithArgs(args, func() error {
-				return cmdDel(args)
+				link, err := netlink.LinkByName("foobar0")
+				Expect(err).NotTo(HaveOccurred())
+				Expect(link.Attrs().Name).To(Equal("foobar0"))
+				Expect(link.Attrs().MTU).To(Equal(1500))
+				return nil
 			})
 			Expect(err).NotTo(HaveOccurred())
-			return nil
 		})
-		Expect(err).NotTo(HaveOccurred())
 
-		// Make sure vlan link has been deleted
-		err = targetNs.Do(func(ns.NetNS) error {
-			defer GinkgoRecover()
+		It(fmt.Sprintf("[%s] creates an vlan link in a non-default namespace with master's MTU", ver), func() {
+			conf := &NetConf{
+				NetConf: types.NetConf{
+					CNIVersion: ver,
+					Name:       "testConfig",
+					Type:       "vlan",
+				},
+				Master: MASTER_NAME,
+				VlanId: 33,
+			}
 
-			link, err := netlink.LinkByName(IFNAME)
-			Expect(err).To(HaveOccurred())
-			Expect(link).To(BeNil())
-			return nil
-		})
-		Expect(err).NotTo(HaveOccurred())
+			// Create vlan in other namespace
+			err := originalNS.Do(func(ns.NetNS) error {
+				defer GinkgoRecover()
 
-		// DEL can be called multiple times, make sure no error is returned
-		// if the device is already removed.
-		err = originalNS.Do(func(ns.NetNS) error {
-			defer GinkgoRecover()
+				m, err := netlink.LinkByName(MASTER_NAME)
+				Expect(err).NotTo(HaveOccurred())
+				err = netlink.LinkSetMTU(m, 1200)
+				Expect(err).NotTo(HaveOccurred())
 
-			err = testutils.CmdDelWithArgs(args, func() error {
-				return cmdDel(args)
-			})
-			Expect(err).NotTo(HaveOccurred())
-			return nil
-		})
-		Expect(err).NotTo(HaveOccurred())
-	})
-
-	It("configures and deconfigures an CNI V4 vlan link with ADD/CHECK/DEL", func() {
-		const IFNAME = "eth0"
-
-		conf := fmt.Sprintf(`{
-    "cniVersion": "0.4.0",
-    "name": "vlanTestv4",
-    "type": "vlan",
-    "master": "%s",
-    "vlanId": 1234,
-    "ipam": {
-        "type": "host-local",
-        "subnet": "10.1.2.0/24"
-    }
-}`, MASTER_NAME)
-
-		targetNs, err := testutils.NewNS()
-		Expect(err).NotTo(HaveOccurred())
-		defer targetNs.Close()
-
-		args := &skel.CmdArgs{
-			ContainerID: "dummy",
-			Netns:       targetNs.Path(),
-			IfName:      IFNAME,
-			StdinData:   []byte(conf),
-		}
-
-		var result *current.Result
-		err = originalNS.Do(func(ns.NetNS) error {
-			defer GinkgoRecover()
-
-			r, _, err := testutils.CmdAddWithArgs(args, func() error {
-				return cmdAdd(args)
+				_, err = createVlan(conf, "foobar0", targetNS)
+				Expect(err).NotTo(HaveOccurred())
+				return nil
 			})
 			Expect(err).NotTo(HaveOccurred())
 
-			result, err = current.GetResult(r)
-			Expect(err).NotTo(HaveOccurred())
+			// Make sure vlan link exists in the target namespace
+			err = targetNS.Do(func(ns.NetNS) error {
+				defer GinkgoRecover()
 
-			Expect(len(result.Interfaces)).To(Equal(1))
-			Expect(result.Interfaces[0].Name).To(Equal(IFNAME))
-			Expect(len(result.IPs)).To(Equal(1))
-			return nil
-		})
-		Expect(err).NotTo(HaveOccurred())
-
-		// Make sure vlan link exists in the target namespace
-		err = targetNs.Do(func(ns.NetNS) error {
-			defer GinkgoRecover()
-
-			link, err := netlink.LinkByName(IFNAME)
-			Expect(err).NotTo(HaveOccurred())
-			Expect(link.Attrs().Name).To(Equal(IFNAME))
-
-			hwaddr, err := net.ParseMAC(result.Interfaces[0].Mac)
-			Expect(err).NotTo(HaveOccurred())
-			Expect(link.Attrs().HardwareAddr).To(Equal(hwaddr))
-
-			addrs, err := netlink.AddrList(link, syscall.AF_INET)
-			Expect(err).NotTo(HaveOccurred())
-			Expect(len(addrs)).To(Equal(1))
-			return nil
-		})
-		Expect(err).NotTo(HaveOccurred())
-
-		// call CmdCheck
-		n := &Net{}
-		err = json.Unmarshal([]byte(conf), &n)
-		Expect(err).NotTo(HaveOccurred())
-
-		n.IPAM, _, err = allocator.LoadIPAMConfig([]byte(conf), "")
-		Expect(err).NotTo(HaveOccurred())
-
-		cniVersion := "0.4.0"
-		newConf, err := buildOneConfig("vlanTestv4", cniVersion, n, result)
-		Expect(err).NotTo(HaveOccurred())
-
-		confString, err := json.Marshal(newConf)
-		Expect(err).NotTo(HaveOccurred())
-
-		args.StdinData = confString
-
-		// CNI Check host-device in the target namespace
-		err = originalNS.Do(func(ns.NetNS) error {
-			defer GinkgoRecover()
-			var err error
-			err = testutils.CmdCheckWithArgs(args, func() error { return cmdCheck(args) })
-			return err
-		})
-		Expect(err).NotTo(HaveOccurred())
-
-		args.StdinData = []byte(conf)
-
-		err = originalNS.Do(func(ns.NetNS) error {
-			defer GinkgoRecover()
-
-			err = testutils.CmdDelWithArgs(args, func() error {
-				return cmdDel(args)
+				link, err := netlink.LinkByName("foobar0")
+				Expect(err).NotTo(HaveOccurred())
+				Expect(link.Attrs().Name).To(Equal("foobar0"))
+				Expect(link.Attrs().MTU).To(Equal(1200))
+				return nil
 			})
 			Expect(err).NotTo(HaveOccurred())
-			return nil
 		})
-		Expect(err).NotTo(HaveOccurred())
 
-		// Make sure vlan link has been deleted
-		err = targetNs.Do(func(ns.NetNS) error {
-			defer GinkgoRecover()
+		It(fmt.Sprintf("[%s] configures and deconfigures a vlan link with ADD/CHECK/DEL", ver), func() {
+			const IFNAME = "eth0"
 
-			link, err := netlink.LinkByName(IFNAME)
-			Expect(err).To(HaveOccurred())
-			Expect(link).To(BeNil())
-			return nil
-		})
-		Expect(err).NotTo(HaveOccurred())
-	})
+			conf := fmt.Sprintf(`{
+			    "cniVersion": "%s",
+			    "name": "vlanTestv4",
+			    "type": "vlan",
+			    "master": "%s",
+			    "vlanId": 1234,
+			    "ipam": {
+				"type": "host-local",
+				"subnet": "10.1.2.0/24",
+				"dataDir": "%s"
+			    }
+			}`, ver, MASTER_NAME, dataDir)
+
+			args := &skel.CmdArgs{
+				ContainerID: "dummy",
+				Netns:       targetNS.Path(),
+				IfName:      IFNAME,
+				StdinData:   []byte(conf),
+			}
+
+			t := newTesterByVersion(ver)
+
+			var result types.Result
+			var macAddress string
+			err := originalNS.Do(func(ns.NetNS) error {
+				defer GinkgoRecover()
+
+				var err error
+				result, _, err = testutils.CmdAddWithArgs(args, func() error {
+					return cmdAdd(args)
+				})
+				Expect(err).NotTo(HaveOccurred())
+
+				macAddress = t.verifyResult(result, IFNAME)
+				return nil
+			})
+			Expect(err).NotTo(HaveOccurred())
+
+			// Make sure vlan link exists in the target namespace
+			err = targetNS.Do(func(ns.NetNS) error {
+				defer GinkgoRecover()
+
+				link, err := netlink.LinkByName(IFNAME)
+				Expect(err).NotTo(HaveOccurred())
+				Expect(link.Attrs().Name).To(Equal(IFNAME))
+
+				if macAddress != "" {
+					hwaddr, err := net.ParseMAC(macAddress)
+					Expect(err).NotTo(HaveOccurred())
+					Expect(link.Attrs().HardwareAddr).To(Equal(hwaddr))
+				}
+
+				addrs, err := netlink.AddrList(link, syscall.AF_INET)
+				Expect(err).NotTo(HaveOccurred())
+				Expect(len(addrs)).To(Equal(1))
+				return nil
+			})
+			Expect(err).NotTo(HaveOccurred())
+
+			// call CmdCheck
+			n := &Net{}
+			err = json.Unmarshal([]byte(conf), &n)
+			Expect(err).NotTo(HaveOccurred())
+
+			n.IPAM, _, err = allocator.LoadIPAMConfig([]byte(conf), "")
+			Expect(err).NotTo(HaveOccurred())
+
+			newConf, err := buildOneConfig("vlanTestv4", ver, n, result)
+			Expect(err).NotTo(HaveOccurred())
+
+			confString, err := json.Marshal(newConf)
+			Expect(err).NotTo(HaveOccurred())
+
+			args.StdinData = confString
+
+			// CNI Check host-device in the target namespace
+			err = originalNS.Do(func(ns.NetNS) error {
+				defer GinkgoRecover()
+				return testutils.CmdCheckWithArgs(args, func() error { return cmdCheck(args) })
+			})
+			if testutils.SpecVersionHasCHECK(ver) {
+				Expect(err).NotTo(HaveOccurred())
+			} else {
+				Expect(err).To(MatchError("config version does not allow CHECK"))
+			}
+
+			args.StdinData = []byte(conf)
 
-	Describe("fails to create vlan link with invalid MTU", func() {
-		conf := `{
-    "cniVersion": "0.3.1",
-    "name": "mynet",
-    "type": "vlan",
-    "master": "%s",
-    "mtu": %d,
-    "ipam": {
-        "type": "host-local",
-        "subnet": "10.1.2.0/24"
-    }
-}`
-		BeforeEach(func() {
-			var err error
 			err = originalNS.Do(func(ns.NetNS) error {
 				defer GinkgoRecover()
 
-				// set master link's MTU to 1500
-				link, err := netlink.LinkByName(MASTER_NAME)
-				Expect(err).NotTo(HaveOccurred())
-				err = netlink.LinkSetMTU(link, 1500)
+				err = testutils.CmdDelWithArgs(args, func() error {
+					return cmdDel(args)
+				})
 				Expect(err).NotTo(HaveOccurred())
+				return nil
+			})
+			Expect(err).NotTo(HaveOccurred())
 
+			// Make sure vlan link has been deleted
+			err = targetNS.Do(func(ns.NetNS) error {
+				defer GinkgoRecover()
+
+				link, err := netlink.LinkByName(IFNAME)
+				Expect(err).To(HaveOccurred())
+				Expect(link).To(BeNil())
+				return nil
+			})
+			Expect(err).NotTo(HaveOccurred())
+
+			// DEL can be called multiple times, make sure no error is returned
+			// if the device is already removed.
+			err = originalNS.Do(func(ns.NetNS) error {
+				defer GinkgoRecover()
+
+				err = testutils.CmdDelWithArgs(args, func() error {
+					return cmdDel(args)
+				})
+				Expect(err).NotTo(HaveOccurred())
 				return nil
 			})
 			Expect(err).NotTo(HaveOccurred())
 		})
 
-		It("fails to create vlan link with greater MTU than master interface", func() {
-			var err error
+		Describe("fails to create vlan link with invalid MTU", func() {
+			const confFmt = `{
+			    "cniVersion": "%s",
+			    "name": "mynet",
+			    "type": "vlan",
+			    "master": "%s",
+			    "mtu": %d,
+			    "ipam": {
+				"type": "host-local",
+				"subnet": "10.1.2.0/24",
+				"dataDir": "%s"
+			    }
+			}`
 
-			args := &skel.CmdArgs{
-				ContainerID: "dummy",
-				Netns:       "/var/run/netns/test",
-				IfName:      "eth0",
-				StdinData:   []byte(fmt.Sprintf(conf, MASTER_NAME, 1600)),
-			}
+			BeforeEach(func() {
+				var err error
+				err = originalNS.Do(func(ns.NetNS) error {
+					defer GinkgoRecover()
 
-			_ = originalNS.Do(func(netNS ns.NetNS) error {
-				defer GinkgoRecover()
+					// set master link's MTU to 1500
+					link, err := netlink.LinkByName(MASTER_NAME)
+					Expect(err).NotTo(HaveOccurred())
+					err = netlink.LinkSetMTU(link, 1500)
+					Expect(err).NotTo(HaveOccurred())
 
-				_, _, err = testutils.CmdAddWithArgs(args, func() error {
-					return cmdAdd(args)
+					return nil
+				})
+				Expect(err).NotTo(HaveOccurred())
+			})
+
+			It(fmt.Sprintf("[%s] fails to create vlan link with greater MTU than master interface", ver), func() {
+				var err error
+
+				args := &skel.CmdArgs{
+					ContainerID: "dummy",
+					Netns:       "/var/run/netns/test",
+					IfName:      "eth0",
+					StdinData:   []byte(fmt.Sprintf(confFmt, ver, MASTER_NAME, 1600, dataDir)),
+				}
+
+				_ = originalNS.Do(func(netNS ns.NetNS) error {
+					defer GinkgoRecover()
+
+					_, _, err = testutils.CmdAddWithArgs(args, func() error {
+						return cmdAdd(args)
+					})
+					Expect(err).To(Equal(fmt.Errorf("invalid MTU 1600, must be [0, master MTU(1500)]")))
+					return nil
+				})
+			})
+
+			It(fmt.Sprintf("[%s] fails to create vlan link with negative MTU", ver), func() {
+				var err error
+
+				args := &skel.CmdArgs{
+					ContainerID: "dummy",
+					Netns:       "/var/run/netns/test",
+					IfName:      "eth0",
+					StdinData:   []byte(fmt.Sprintf(confFmt, ver, MASTER_NAME, -100, dataDir)),
+				}
+
+				_ = originalNS.Do(func(netNS ns.NetNS) error {
+					defer GinkgoRecover()
+
+					_, _, err = testutils.CmdAddWithArgs(args, func() error {
+						return cmdAdd(args)
+					})
+					Expect(err).To(Equal(fmt.Errorf("invalid MTU -100, must be [0, master MTU(1500)]")))
+					return nil
 				})
-				Expect(err).To(Equal(fmt.Errorf("invalid MTU 1600, must be [0, master MTU(1500)]")))
-				return nil
 			})
 		})
-
-		It("fails to create vlan link with negative MTU", func() {
-			var err error
-
-			args := &skel.CmdArgs{
-				ContainerID: "dummy",
-				Netns:       "/var/run/netns/test",
-				IfName:      "eth0",
-				StdinData:   []byte(fmt.Sprintf(conf, MASTER_NAME, -100)),
-			}
-
-			_ = originalNS.Do(func(netNS ns.NetNS) error {
-				defer GinkgoRecover()
-
-				_, _, err = testutils.CmdAddWithArgs(args, func() error {
-					return cmdAdd(args)
-				})
-				Expect(err).To(Equal(fmt.Errorf("invalid MTU -100, must be [0, master MTU(1500)]")))
-				return nil
-			})
-		})
-	})
+	}
 })

plugins/main/windows/win-bridge/README.md

@@ -1,5 +1,5 @@
 
 This document has moved to the [containernetworking/cni.dev](https://github.com/containernetworking/cni.dev) repo.
 
-You can find it online here: https://cni.dev/plugins/main/win-bridge/
+You can find it online here: https://cni.dev/plugins/current/main/win-bridge/
 

plugins/main/windows/win-bridge/sample-v1.conf

@@ -0,0 +1,42 @@
+{
+  "name": "cbr0",
+  "type": "win-bridge",
+  "dns": {
+    "nameservers": [
+      "11.0.0.10"
+    ],
+    "search": [
+      "svc.cluster.local"
+    ]
+  },
+  "policies": [
+    {
+      "name": "EndpointPolicy",
+      "value": {
+        "Type": "OutBoundNAT",
+        "ExceptionList": [
+          "192.168.0.0/16",
+          "11.0.0.0/8",
+          "10.137.196.0/23"
+        ]
+      }
+    },
+    {
+      "name": "EndpointPolicy",
+      "value": {
+        "Type": "ROUTE",
+        "DestinationPrefix": "11.0.0.0/8",
+        "NeedEncap": true
+      }
+    },
+    {
+      "name": "EndpointPolicy",
+      "value": {
+        "Type": "ROUTE",
+        "DestinationPrefix": "10.137.198.27/32",
+        "NeedEncap": true
+      }
+    }
+  ],
+  "loopbackDSR": true
+}

plugins/main/windows/win-bridge/sample-v2.conf

@@ -0,0 +1,52 @@
+{
+    "name":"cbr0",
+    "type":"flannel",
+    "delegate":{
+        "apiVersion":2,
+        "type":"win-bridge",
+        "dns":{
+            "nameservers":[
+                "11.0.0.10"
+            ],
+            "search":[
+                "svc.cluster.local"
+            ]
+        },
+        "policies":[
+            {
+                "name":"EndpointPolicy",
+                "value":{
+                    "Type":"OutBoundNAT",
+                    "Settings":{
+                        "Exceptions":[
+                            "192.168.0.0/16",
+                            "11.0.0.0/8",
+                            "10.137.196.0/23"
+                        ]
+                    }
+                }
+            },
+            {
+                "name":"EndpointPolicy",
+                "value":{
+                    "Type":"SDNRoute",
+                    "Settings":{
+                        "DestinationPrefix":"11.0.0.0/8",
+                        "NeedEncap":true
+                    }
+                }
+            },
+            {
+                "name":"EndpointPolicy",
+                "value":{
+                    "Type":"SDNRoute",
+                    "Settings":{
+                        "DestinationPrefix":"10.137.198.27/32",
+                        "NeedEncap":true
+                    }
+                }
+            }
+        ],
+        "loopbackDSR":true
+    }
+}

plugins/main/windows/win-bridge/sample.conf

@@ -1,45 +0,0 @@
-{
-    "name":"cbr0",
-    "type":"flannel",
-    "delegate":{
-        "type":"win-bridge",
-        "dns":{
-            "nameservers":[
-                "11.0.0.10"
-            ],
-            "search":[
-                "svc.cluster.local"
-            ]
-        },
-        "policies":[
-            {
-                "name":"EndpointPolicy",
-                "value":{
-                    "Type":"OutBoundNAT",
-                    "ExceptionList":[
-                        "192.168.0.0/16",
-                        "11.0.0.0/8",
-                        "10.137.196.0/23"
-                    ]
-                }
-            },
-            {
-                "name":"EndpointPolicy",
-                "value":{
-                    "Type":"ROUTE",
-                    "DestinationPrefix":"11.0.0.0/8",
-                    "NeedEncap":true
-                }
-            },
-            {
-                "name":"EndpointPolicy",
-                "value":{
-                    "Type":"ROUTE",
-                    "DestinationPrefix":"10.137.198.27/32",
-                    "NeedEncap":true
-                }
-            }
-        ],
-        "loopbackDSR": true
-    }
-}

plugins/main/windows/win-bridge/win-bridge_windows.go

@@ -25,7 +25,7 @@ import (
 
 	"github.com/containernetworking/cni/pkg/skel"
 	"github.com/containernetworking/cni/pkg/types"
-	"github.com/containernetworking/cni/pkg/types/current"
+	current "github.com/containernetworking/cni/pkg/types/100"
 	"github.com/containernetworking/cni/pkg/version"
 
 	"github.com/containernetworking/plugins/pkg/errors"
@@ -55,21 +55,22 @@ func loadNetConf(bytes []byte) (*NetConf, string, error) {
 	return n, n.CNIVersion, nil
 }
 
-func ProcessEndpointArgs(args *skel.CmdArgs, n *NetConf) (*hns.EndpointInfo, error) {
+func processEndpointArgs(args *skel.CmdArgs, n *NetConf) (*hns.EndpointInfo, error) {
 	epInfo := new(hns.EndpointInfo)
 	epInfo.NetworkName = n.Name
 	epInfo.EndpointName = hns.ConstructEndpointName(args.ContainerID, args.Netns, epInfo.NetworkName)
-	// It's not necessary to have have an IPAM in windows as hns can provide IP/GW
+
+	// it's not necessary to have have an IPAM in windows as HNS can provide IP/GW
 	if n.IPAM.Type != "" {
 		r, err := ipam.ExecAdd(n.IPAM.Type, args.StdinData)
 		if err != nil {
-			return nil, errors.Annotatef(err, "error while ipam.ExecAdd")
+			return nil, errors.Annotatef(err, "error while executing IPAM addition")
 		}
 
-		// Convert whatever the IPAM result was into the current Result type
+		// convert whatever the IPAM result was into the current result
 		result, err := current.NewResultFromResult(r)
 		if err != nil {
-			return nil, errors.Annotatef(err, "error while NewResultFromResult")
+			return nil, errors.Annotatef(err, "error while converting the result from IPAM addition")
 		} else {
 			if len(result.IPs) == 0 {
 				return nil, fmt.Errorf("IPAM plugin return is missing IP config")
@@ -81,12 +82,11 @@ func ProcessEndpointArgs(args *skel.CmdArgs, n *NetConf) (*hns.EndpointInfo, err
 			epInfo.Gateway[len(epInfo.Gateway)-1] += 2
 		}
 	}
-	// NAT based on the the configured cluster network
-	if len(n.IPMasqNetwork) != 0 {
-		n.ApplyOutboundNatPolicy(n.IPMasqNetwork)
-	}
 
-	// Add HostPort mapping if any present
+	// configure sNAT exception
+	n.ApplyOutboundNatPolicy(n.IPMasqNetwork)
+
+	// add port mapping if any present
 	n.ApplyPortMappingPolicy(n.RuntimeConfig.PortMaps)
 
 	epInfo.DNS = n.GetDNS()
@@ -98,39 +98,37 @@ func cmdHnsAdd(args *skel.CmdArgs, n *NetConf) (*current.Result, error) {
 	networkName := n.Name
 	hnsNetwork, err := hcsshim.GetHNSNetworkByName(networkName)
 	if err != nil {
-		return nil, errors.Annotatef(err, "error while GETHNSNewtorkByName(%s)", networkName)
+		return nil, errors.Annotatef(err, "error while getting network %v", networkName)
 	}
-
 	if hnsNetwork == nil {
-		return nil, fmt.Errorf("network %v not found", networkName)
+		return nil, fmt.Errorf("network %v is not found", networkName)
 	}
-
 	if !strings.EqualFold(hnsNetwork.Type, "L2Bridge") && !strings.EqualFold(hnsNetwork.Type, "L2Tunnel") {
-		return nil, fmt.Errorf("network %v is of an unexpected type: %v", networkName, hnsNetwork.Type)
+		return nil, fmt.Errorf("network %v is of unexpected type: %v", networkName, hnsNetwork.Type)
 	}
 
 	epName := hns.ConstructEndpointName(args.ContainerID, args.Netns, n.Name)
-	hnsEndpoint, err := hns.ProvisionEndpoint(epName, hnsNetwork.Id, args.ContainerID, args.Netns, func() (*hcsshim.HNSEndpoint, error) {
-		epInfo, err := ProcessEndpointArgs(args, n)
-		epInfo.NetworkId = hnsNetwork.Id
+	hnsEndpoint, err := hns.AddHnsEndpoint(epName, hnsNetwork.Id, args.ContainerID, args.Netns, func() (*hcsshim.HNSEndpoint, error) {
+		epInfo, err := processEndpointArgs(args, n)
 		if err != nil {
-			return nil, errors.Annotatef(err, "error while ProcessEndpointArgs")
+			return nil, errors.Annotate(err, "error while processing endpoint args")
 		}
+		epInfo.NetworkId = hnsNetwork.Id
+
 		hnsEndpoint, err := hns.GenerateHnsEndpoint(epInfo, &n.NetConf)
 		if err != nil {
-			return nil, errors.Annotatef(err, "error while GenerateHnsEndpoint")
+			return nil, errors.Annotate(err, "error while generating HNSEndpoint")
 		}
 		return hnsEndpoint, nil
 	})
 	if err != nil {
-		return nil, errors.Annotatef(err, "error while ProvisionEndpoint(%v,%v,%v)", epName, hnsNetwork.Id, args.ContainerID)
+		return nil, errors.Annotate(err, "error while adding HNSEndpoint")
 	}
 
-	result, err := hns.ConstructResult(hnsNetwork, hnsEndpoint)
+	result, err := hns.ConstructHnsResult(hnsNetwork, hnsEndpoint)
 	if err != nil {
-		return nil, errors.Annotatef(err, "error while constructResult")
+		return nil, errors.Annotate(err, "error while constructing HNSEndpoint addition result")
 	}
-
 	return result, nil
 }
 
@@ -138,48 +136,44 @@ func cmdHcnAdd(args *skel.CmdArgs, n *NetConf) (*current.Result, error) {
 	networkName := n.Name
 	hcnNetwork, err := hcn.GetNetworkByName(networkName)
 	if err != nil {
-		return nil, errors.Annotatef(err, "error while GetNetworkByName(%s)", networkName)
+		return nil, errors.Annotatef(err, "error while getting network %v", networkName)
 	}
-
 	if hcnNetwork == nil {
-		return nil, fmt.Errorf("network %v not found", networkName)
+		return nil, fmt.Errorf("network %v is not found", networkName)
 	}
-
 	if hcnNetwork.Type != hcn.L2Bridge && hcnNetwork.Type != hcn.L2Tunnel {
 		return nil, fmt.Errorf("network %v is of unexpected type: %v", networkName, hcnNetwork.Type)
 	}
 
 	epName := hns.ConstructEndpointName(args.ContainerID, args.Netns, n.Name)
-
 	hcnEndpoint, err := hns.AddHcnEndpoint(epName, hcnNetwork.Id, args.Netns, func() (*hcn.HostComputeEndpoint, error) {
-		epInfo, err := ProcessEndpointArgs(args, n)
+		epInfo, err := processEndpointArgs(args, n)
 		if err != nil {
-			return nil, errors.Annotatef(err, "error while ProcessEndpointArgs")
+			return nil, errors.Annotate(err, "error while processing endpoint args")
 		}
 		epInfo.NetworkId = hcnNetwork.Id
 
 		hcnEndpoint, err := hns.GenerateHcnEndpoint(epInfo, &n.NetConf)
 		if err != nil {
-			return nil, errors.Annotatef(err, "error while GenerateHcnEndpoint")
+			return nil, errors.Annotate(err, "error while generating HostComputeEndpoint")
 		}
 		return hcnEndpoint, nil
 	})
 	if err != nil {
-		return nil, errors.Annotatef(err, "error while AddHcnEndpoint(%v,%v,%v)", epName, hcnNetwork.Id, args.Netns)
+		return nil, errors.Annotate(err, "error while adding HostComputeEndpoint")
 	}
 
 	result, err := hns.ConstructHcnResult(hcnNetwork, hcnEndpoint)
 	if err != nil {
-		return nil, errors.Annotatef(err, "error while ConstructHcnResult")
+		return nil, errors.Annotate(err, "error while constructing HostComputeEndpoint addition result")
 	}
-
 	return result, nil
 }
 
 func cmdAdd(args *skel.CmdArgs) error {
 	n, cniVersion, err := loadNetConf(args.StdinData)
 	if err != nil {
-		return errors.Annotate(err, "error while loadNetConf")
+		return err
 	}
 
 	var result *current.Result
@@ -188,15 +182,11 @@ func cmdAdd(args *skel.CmdArgs) error {
 	} else {
 		result, err = cmdHnsAdd(args, n)
 	}
-
 	if err != nil {
 		ipam.ExecDel(n.IPAM.Type, args.StdinData)
-		return errors.Annotate(err, "error while executing ADD command")
+		return err
 	}
 
-	if result == nil {
-		return fmt.Errorf("result for ADD not populated correctly")
-	}
 	return types.PrintResult(result, cniVersion)
 }
 
@@ -215,9 +205,8 @@ func cmdDel(args *skel.CmdArgs) error {
 
 	if n.ApiVersion == 2 {
 		return hns.RemoveHcnEndpoint(epName)
-	} else {
-		return hns.DeprovisionEndpoint(epName, args.Netns, args.ContainerID)
 	}
+	return hns.RemoveHnsEndpoint(epName, args.Netns, args.ContainerID)
 }
 
 func cmdCheck(_ *skel.CmdArgs) error {
@@ -226,5 +215,5 @@ func cmdCheck(_ *skel.CmdArgs) error {
 }
 
 func main() {
-	skel.PluginMain(cmdAdd, cmdCheck, cmdDel, version.PluginSupports("0.1.0", "0.2.0", "0.3.0"), bv.BuildString("win-bridge"))
+	skel.PluginMain(cmdAdd, cmdCheck, cmdDel, version.All, bv.BuildString("win-bridge"))
 }

plugins/main/windows/win-overlay/README.md

@@ -1,5 +1,5 @@
 
 This document has moved to the [containernetworking/cni.dev](https://github.com/containernetworking/cni.dev) repo.
 
-You can find it online here: https://cni.dev/plugins/main/win-overlay/
+You can find it online here: https://cni.dev/plugins/current/main/win-overlay/
 

plugins/main/windows/win-overlay/sample-v2.conf

@@ -0,0 +1,46 @@
+{
+  "cniVersion": "0.2.0",
+  "name": "OVNKubernetesHybridOverlayNetwork",
+  "type": "win-overlay",
+  "ipam": {
+    "type": "host-local",
+    "subnet": "10.132.0.0/24"
+  },
+  "apiVersion": 2,
+  "capabilities": {
+    "portMappings": true,
+    "dns": true
+  },
+  "policies": [
+    {
+      "name": "EndpointPolicy",
+      "value": {
+        "Type": "OutBoundNAT",
+        "Settings": {
+          "Exceptions": [
+            "172.30.0.0/16"
+          ]
+        }
+      }
+    },
+    {
+      "name": "EndpointPolicy",
+      "value": {
+        "Type": "SDNRoute",
+        "Settings": {
+          "DestinationPrefix": "172.30.0.0/16",
+          "NeedEncap": true
+        }
+      }
+    },
+    {
+      "name": "EndpointPolicy",
+      "value": {
+        "Type": "ProviderAddress",
+        "Settings": {
+          "ProviderAddress": "10.0.133.170"
+        }
+      }
+    }
+  ]
+}

plugins/main/windows/win-overlay/sample.conf

@@ -1,36 +1,33 @@
-{
-    "cniVersion":"0.2.0",
-    "name":"vxlan0",
-    "type":"flannel",
-    "delegate":{
-        "type":"win-overlay",
-        "dns":{
-            "nameservers":[
-                "11.0.0.10"
-            ],
-            "search":[
-                "svc.cluster.local"
-            ]
-        },
-        "policies":[
-            {
-                "name":"EndpointPolicy",
-                "value":{
-                    "Type":"OutBoundNAT",
-                    "ExceptionList":[
-                        "192.168.0.0/16",
-                        "11.0.0.0/8"
-                    ]
-                }
-            },
-            {
-                "name":"EndpointPolicy",
-                "value":{
-                    "Type":"ROUTE",
-                    "DestinationPrefix":"11.0.0.0/8",
-                    "NeedEncap":true
-                }
-            }
-        ]
-    }
-}
+{
+  "cniVersion": "0.2.0",
+  "name": "vxlan0",
+  "type": "win-overlay",
+  "dns": {
+    "nameservers": [
+      "11.0.0.10"
+    ],
+    "search": [
+      "svc.cluster.local"
+    ]
+  },
+  "policies": [
+    {
+      "name": "EndpointPolicy",
+      "value": {
+        "Type": "OutBoundNAT",
+        "ExceptionList": [
+          "192.168.0.0/16",
+          "11.0.0.0/8"
+        ]
+      }
+    },
+    {
+      "name": "EndpointPolicy",
+      "value": {
+        "Type": "ROUTE",
+        "DestinationPrefix": "11.0.0.0/8",
+        "NeedEncap": true
+      }
+    }
+  ]
+}

plugins/main/windows/win-overlay/win-overlay_windows.go

@@ -17,14 +17,16 @@ package main
 import (
 	"encoding/json"
 	"fmt"
+	"net"
 	"runtime"
 	"strings"
 
 	"github.com/Microsoft/hcsshim"
 
+	"github.com/Microsoft/hcsshim/hcn"
 	"github.com/containernetworking/cni/pkg/skel"
 	"github.com/containernetworking/cni/pkg/types"
-	"github.com/containernetworking/cni/pkg/types/current"
+	current "github.com/containernetworking/cni/pkg/types/100"
 	"github.com/containernetworking/cni/pkg/version"
 
 	"github.com/containernetworking/plugins/pkg/errors"
@@ -55,16 +57,106 @@ func loadNetConf(bytes []byte) (*NetConf, string, error) {
 	return n, n.CNIVersion, nil
 }
 
-func cmdAdd(args *skel.CmdArgs) error {
-	success := false
-	n, cniVersion, err := loadNetConf(args.StdinData)
-	if err != nil {
-		return errors.Annotate(err, "error while loadNetConf")
+func processEndpointArgs(args *skel.CmdArgs, n *NetConf) (*hns.EndpointInfo, error) {
+	epInfo := new(hns.EndpointInfo)
+	epInfo.NetworkName = n.Name
+	epInfo.EndpointName = hns.ConstructEndpointName(args.ContainerID, args.Netns, epInfo.NetworkName)
+
+	if n.IPAM.Type != "" {
+		r, err := ipam.ExecAdd(n.IPAM.Type, args.StdinData)
+		if err != nil {
+			return nil, errors.Annotatef(err, "error while executing IPAM addition")
+		}
+
+		// convert whatever the IPAM result was into the current result
+		result, err := current.NewResultFromResult(r)
+		if err != nil {
+			return nil, errors.Annotatef(err, "error while converting the result from IPAM addition")
+		}
+		if len(result.IPs) == 0 {
+			return nil, fmt.Errorf("IPAM plugin return is missing IP config")
+		}
+		epInfo.IpAddress = result.IPs[0].Address.IP.To4()
+		if epInfo.IpAddress == nil {
+			return nil, fmt.Errorf("IPAM plugin return is missing valid IP Address")
+		}
+		epInfo.MacAddress = fmt.Sprintf("%v-%02x-%02x-%02x-%02x", n.EndpointMacPrefix, epInfo.IpAddress[0], epInfo.IpAddress[1], epInfo.IpAddress[2], epInfo.IpAddress[3])
+
 	}
+	epInfo.DNS = n.GetDNS()
+	if n.LoopbackDSR {
+		n.ApplyLoopbackDSRPolicy(&epInfo.IpAddress)
+	}
+	return epInfo, nil
+}
+
+func cmdHcnAdd(args *skel.CmdArgs, n *NetConf) (*current.Result, error) {
+	if len(n.EndpointMacPrefix) != 0 {
+		if len(n.EndpointMacPrefix) != 5 || n.EndpointMacPrefix[2] != '-' {
+			return nil, fmt.Errorf("endpointMacPrefix [%v] is invalid, value must be of the format xx-xx", n.EndpointMacPrefix)
+		}
+	} else {
+		n.EndpointMacPrefix = "0E-2A"
+	}
+
+	networkName := n.Name
+	hnsNetwork, err := hcsshim.GetHNSNetworkByName(networkName)
+	hcnNetwork, err := hcn.GetNetworkByName(networkName)
+	if err != nil {
+		return nil, errors.Annotatef(err, "error while hcn.GetNetworkByName(%s)", networkName)
+	}
+	if hcnNetwork == nil {
+		return  nil, fmt.Errorf("network %v is not found", networkName)
+	}
+	if hnsNetwork == nil {
+		return nil, fmt.Errorf("network %v not found", networkName)
+	}
+
+	if !strings.EqualFold(string (hcnNetwork.Type), "Overlay") {
+		return nil, fmt.Errorf("network %v is of an unexpected type: %v", networkName, hcnNetwork.Type)
+	}
+
+	epName := hns.ConstructEndpointName(args.ContainerID, args.Netns, n.Name)
+
+	hcnEndpoint, err := hns.AddHcnEndpoint(epName, hcnNetwork.Id, args.Netns, func() (*hcn.HostComputeEndpoint, error) {
+		epInfo, err := processEndpointArgs(args, n)
+		if err != nil {
+			return nil, errors.Annotate(err, "error while processing endpoint args")
+		}
+		epInfo.NetworkId = hcnNetwork.Id
+		gatewayAddr := net.ParseIP(hnsNetwork.Subnets[0].GatewayAddress)
+		epInfo.Gateway = gatewayAddr.To4()
+		n.ApplyDefaultPAPolicy(hnsNetwork.ManagementIP)
+		if n.IPMasq {
+			n.ApplyOutboundNatPolicy(hnsNetwork.Subnets[0].AddressPrefix)
+		}
+		hcnEndpoint, err := hns.GenerateHcnEndpoint(epInfo, &n.NetConf)
+
+		if err != nil {
+			return nil, errors.Annotate(err, "error while generating HostComputeEndpoint")
+		}
+		return hcnEndpoint, nil
+	})
+	if err != nil {
+		return nil, errors.Annotate(err, "error while adding HostComputeEndpoint")
+	}
+
+	result, err := hns.ConstructHcnResult(hcnNetwork, hcnEndpoint)
+
+	if err != nil {
+		ipam.ExecDel(n.IPAM.Type, args.StdinData)
+		return nil, errors.Annotate(err, "error while constructing HostComputeEndpoint addition result")
+	}
+
+	return result, nil
+
+}
+func cmdHnsAdd(args *skel.CmdArgs, n *NetConf) (*current.Result, error) {
+	success := false
 
 	if len(n.EndpointMacPrefix) != 0 {
 		if len(n.EndpointMacPrefix) != 5 || n.EndpointMacPrefix[2] != '-' {
-			return fmt.Errorf("endpointMacPrefix [%v] is invalid, value must be of the format xx-xx", n.EndpointMacPrefix)
+			return nil, fmt.Errorf("endpointMacPrefix [%v] is invalid, value must be of the format xx-xx", n.EndpointMacPrefix)
 		}
 	} else {
 		n.EndpointMacPrefix = "0E-2A"
@@ -73,20 +165,20 @@ func cmdAdd(args *skel.CmdArgs) error {
 	networkName := n.Name
 	hnsNetwork, err := hcsshim.GetHNSNetworkByName(networkName)
 	if err != nil {
-		return errors.Annotatef(err, "error while GETHNSNewtorkByName(%s)", networkName)
+		return nil, errors.Annotatef(err, "error while GETHNSNewtorkByName(%s)", networkName)
 	}
 
 	if hnsNetwork == nil {
-		return fmt.Errorf("network %v not found", networkName)
+		return nil, fmt.Errorf("network %v not found", networkName)
 	}
 
 	if !strings.EqualFold(hnsNetwork.Type, "Overlay") {
-		return fmt.Errorf("network %v is of an unexpected type: %v", networkName, hnsNetwork.Type)
+		return nil, fmt.Errorf("network %v is of an unexpected type: %v", networkName, hnsNetwork.Type)
 	}
 
 	epName := hns.ConstructEndpointName(args.ContainerID, args.Netns, n.Name)
 
-	hnsEndpoint, err := hns.ProvisionEndpoint(epName, hnsNetwork.Id, args.ContainerID, args.Netns, func() (*hcsshim.HNSEndpoint, error) {
+	hnsEndpoint, err := hns.AddHnsEndpoint(epName, hnsNetwork.Id, args.ContainerID, args.Netns, func() (*hcsshim.HNSEndpoint, error) {
 		// run the IPAM plugin and get back the config to apply
 		r, err := ipam.ExecAdd(n.IPAM.Type, args.StdinData)
 		if err != nil {
@@ -119,7 +211,7 @@ func cmdAdd(args *skel.CmdArgs) error {
 
 		result.DNS = n.GetDNS()
 		if n.LoopbackDSR {
-			n.ApplyLoopbackDSR(&ipAddr)
+			n.ApplyLoopbackDSRPolicy(&ipAddr)
 		}
 		hnsEndpoint := &hcsshim.HNSEndpoint{
 			Name:           epName,
@@ -129,7 +221,7 @@ func cmdAdd(args *skel.CmdArgs) error {
 			GatewayAddress: gw,
 			IPAddress:      ipAddr,
 			MacAddress:     macAddr,
-			Policies:       n.MarshalPolicies(),
+			Policies:       n.GetHNSEndpointPolicies(),
 		}
 
 		return hnsEndpoint, nil
@@ -140,15 +232,34 @@ func cmdAdd(args *skel.CmdArgs) error {
 		}
 	}()
 	if err != nil {
-		return errors.Annotatef(err, "error while ProvisionEndpoint(%v,%v,%v)", epName, hnsNetwork.Id, args.ContainerID)
+		return nil, errors.Annotatef(err, "error while AddHnsEndpoint(%v,%v,%v)", epName, hnsNetwork.Id, args.ContainerID)
 	}
 
-	result, err := hns.ConstructResult(hnsNetwork, hnsEndpoint)
+	result, err := hns.ConstructHnsResult(hnsNetwork, hnsEndpoint)
 	if err != nil {
-		return errors.Annotatef(err, "error while constructResult")
+		return nil, errors.Annotatef(err, "error while constructResult")
 	}
 
 	success = true
+	return result, nil
+}
+func cmdAdd(args *skel.CmdArgs) error {
+	n, cniVersion, err := loadNetConf(args.StdinData)
+	if err != nil {
+		return err
+	}
+
+	var result *current.Result
+	if n.ApiVersion == 2 {
+		result, err = cmdHcnAdd(args, n)
+	} else {
+		result, err = cmdHnsAdd(args, n)
+	}
+	if err != nil {
+		ipam.ExecDel(n.IPAM.Type, args.StdinData)
+		return err
+	}
+
 	return types.PrintResult(result, cniVersion)
 }
 
@@ -158,13 +269,17 @@ func cmdDel(args *skel.CmdArgs) error {
 		return err
 	}
 
-	if err := ipam.ExecDel(n.IPAM.Type, args.StdinData); err != nil {
-		return err
+	if n.IPAM.Type != "" {
+		if err := ipam.ExecDel(n.IPAM.Type, args.StdinData); err != nil {
+			return err
+		}
 	}
-
 	epName := hns.ConstructEndpointName(args.ContainerID, args.Netns, n.Name)
 
-	return hns.DeprovisionEndpoint(epName, args.Netns, args.ContainerID)
+	if n.ApiVersion == 2 {
+		return hns.RemoveHcnEndpoint(epName)
+	}
+	return hns.RemoveHnsEndpoint(epName, args.Netns, args.ContainerID)
 }
 
 func cmdCheck(_ *skel.CmdArgs) error {
@@ -173,5 +288,5 @@ func cmdCheck(_ *skel.CmdArgs) error {
 }
 
 func main() {
-	skel.PluginMain(cmdAdd, cmdCheck, cmdDel, version.PluginSupports("0.1.0", "0.2.0", "0.3.0"), bv.BuildString("win-overlay"))
+	skel.PluginMain(cmdAdd, cmdCheck, cmdDel, version.All, bv.BuildString("win-overlay"))
 }

plugins/meta/bandwidth/README.md

@@ -1,5 +1,5 @@
 
 This document has moved to the [containernetworking/cni.dev](https://github.com/containernetworking/cni.dev) repo.
 
-You can find it online here: https://cni.dev/plugins/meta/bandwidth/
+You can find it online here: https://cni.dev/plugins/current/meta/bandwidth/