Merge pull request #705 from devbv/master

portmap: fix bug that new udp connection deletes all existing conntrack entries

.github/actions/retest-action/Dockerfile

@@ -0,0 +1,7 @@
+FROM alpine:3.10
+
+RUN apk add --no-cache curl jq
+
+COPY entrypoint.sh /entrypoint.sh
+
+ENTRYPOINT ["/entrypoint.sh"]

.github/actions/retest-action/action.yml

@@ -0,0 +1,11 @@
+name: 'Re-Test'
+description: 'Re-Runs the last workflow for a PR'
+inputs:
+  token:
+    description: 'GitHub API Token'
+    required: true
+runs:
+  using: 'docker'
+  image: 'Dockerfile'
+  env:
+    GITHUB_TOKEN: ${{ inputs.token }}

.github/actions/retest-action/entrypoint.sh

@@ -0,0 +1,45 @@
+#!/bin/sh
+
+set -ex
+
+if ! jq -e '.issue.pull_request' ${GITHUB_EVENT_PATH}; then
+    echo "Not a PR... Exiting."
+    exit 0
+fi
+
+if [ "$(jq -r '.comment.body' ${GITHUB_EVENT_PATH})" != "/retest" ]; then
+    echo "Nothing to do... Exiting."
+    exit 0
+fi
+
+PR_URL=$(jq -r '.issue.pull_request.url' ${GITHUB_EVENT_PATH})
+
+curl --request GET \
+    --url "${PR_URL}" \
+    --header "authorization: Bearer ${GITHUB_TOKEN}" \
+    --header "content-type: application/json" > pr.json
+
+ACTOR=$(jq -r '.user.login' pr.json)
+BRANCH=$(jq -r '.head.ref' pr.json)
+
+curl --request GET \
+    --url "https://api.github.com/repos/${GITHUB_REPOSITORY}/actions/runs?event=pull_request&actor=${ACTOR}&branch=${BRANCH}" \
+    --header "authorization: Bearer ${GITHUB_TOKEN}" \
+    --header "content-type: application/json" | jq '.workflow_runs | max_by(.run_number)' > run.json
+
+RERUN_URL=$(jq -r '.rerun_url' run.json)
+
+curl --request POST \
+    --url "${RERUN_URL}" \
+    --header "authorization: Bearer ${GITHUB_TOKEN}" \
+    --header "content-type: application/json"
+
+
+REACTION_URL="$(jq -r '.comment.url' ${GITHUB_EVENT_PATH})/reactions"
+
+curl --request POST \
+    --url "${REACTION_URL}" \
+    --header "authorization: Bearer ${GITHUB_TOKEN}" \
+    --header "accept: application/vnd.github.squirrel-girl-preview+json" \
+    --header "content-type: application/json" \
+    --data '{ "content" : "rocket" }'

.github/workflows/commands.yml

@@ -0,0 +1,17 @@
+name: commands
+on:
+  issue_comment:
+    types: [created]
+
+jobs:
+  retest:
+    if: github.repository == 'containernetworking/plugins'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v2
+
+      - name: Re-Test Action
+        uses: ./.github/actions/retest-action
+        with:
+          token: ${{ secrets.REPO_ACCESS_TOKEN }}

.github/workflows/test.yaml

@@ -0,0 +1,80 @@
+---
+name: test
+
+on: ["push", "pull_request"]
+
+env:
+  GO_VERSION: "1.17"
+  LINUX_ARCHES: "amd64 386 arm arm64 s390x mips64le ppc64le"
+
+jobs:
+  build:
+    name: Build all linux architectures
+    runs-on: ubuntu-latest
+    steps:
+      - name: setup go
+        uses: actions/setup-go@v2
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - uses: actions/checkout@v2
+
+      - name: Build on all supported architectures
+        run: |
+          set -e
+          for arch in ${LINUX_ARCHES}; do
+            echo "Building for arch $arch"
+            GOARCH=$arch ./build_linux.sh
+            rm bin/*
+          done
+
+  test-linux:
+    name: Run tests on Linux amd64
+    runs-on: ubuntu-latest
+    steps:
+      - name: Install kernel module
+        run: |
+          sudo apt-get update
+          sudo apt-get install linux-modules-extra-$(uname -r)
+      - name: Install nftables
+        run: sudo apt-get install nftables
+
+      - name: setup go
+        uses: actions/setup-go@v2
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: Set up Go for root
+        run: |
+          sudo ln -sf `which go` `sudo which go` || true
+          sudo go version
+      - uses: actions/checkout@v2
+
+      - name: Install test binaries
+        env:
+          GO111MODULE: off
+        run: |
+          go get github.com/containernetworking/cni/cnitool
+          go get github.com/mattn/goveralls
+          go get github.com/modocache/gover
+
+      - name: test
+        run: PATH=$PATH:$(go env GOPATH)/bin COVERALLS=1 ./test_linux.sh
+
+      - name: Send coverage to coveralls
+        env:
+          COVERALLS_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          PATH=$PATH:$(go env GOPATH)/bin
+          gover
+          goveralls -coverprofile=gover.coverprofile -service=github
+
+  test-win:
+    name: Build and run tests on Windows
+    runs-on: windows-latest
+    steps:
+      - name: setup go
+        uses: actions/setup-go@v2
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - uses: actions/checkout@v2
+      - name: test
+        run: bash ./test_windows.sh

.travis.yml

@@ -1,52 +0,0 @@
-language: go
-sudo: required
-dist: xenial
-
-go:
-  - 1.13.x
-  - 1.14.x
-
-env:
-  global:
-  - PATH=$GOROOT/bin:$GOPATH/bin:$PATH
-  - CGO_ENABLED=0
-  matrix:
-  - TARGET=386
-  - TARGET=amd64
-  - TARGET=arm
-  - TARGET=arm64
-  - TARGET=ppc64le
-  - TARGET=s390x
-  - TARGET=mips64le
-
-matrix:
-  fast_finish: true
-  include:
-    - os: windows
-      env: TARGET=amd64
-      go: 1.13.x
-    - os: windows
-      env: TARGET=amd64
-      go: 1.14.x
-
-install:
-  - go get github.com/onsi/ginkgo/ginkgo
-  - go get github.com/containernetworking/cni/cnitool
-  - go get golang.org/x/tools/cmd/cover
-  - go get github.com/modocache/gover
-  - go get github.com/mattn/goveralls
-  - go mod vendor
-
-script:
- - |
-   if [ "${TARGET}" == "amd64" ]; then
-     GOARCH="${TARGET}" ./test_${TRAVIS_OS_NAME}.sh
-   else
-     GOARCH="${TARGET}" ./build_linux.sh
-   fi
-
-notifications:
-  email: false
-
-git:
-  depth: 9999999

OWNERS.md

@@ -1,7 +1,6 @@
 # Owners
 This is the official list of the CNI network plugins owners:
 - Bruce Ma <brucema19901024@gmail.com> (@mars1024)
-- Bryan Boreham <bryan@weave.works> (@bboreham)
 - Casey Callendrello <cdc@redhat.com> (@squeed)
 - Dan Williams <dcbw@redhat.com> (@dcbw)
 - Gabe Rosenhouse <grosenhouse@pivotal.io> (@rosenhouse)

README.md

@@ -1,7 +1,7 @@
-[![Build Status](https://travis-ci.org/containernetworking/plugins.svg?branch=master)](https://travis-ci.org/containernetworking/plugins)
+[![test](https://github.com/containernetworking/plugins/actions/workflows/test.yaml/badge.svg)](https://github.com/containernetworking/plugins/actions/workflows/test.yaml?query=branch%3Amaster)
 
-# plugins
-Some CNI network plugins, maintained by the containernetworking team. For more information, see the individual READMEs.
+# Plugins
+Some CNI network plugins, maintained by the containernetworking team. For more information, see the [CNI website](https://www.cni.dev).
 
 Read [CONTRIBUTING](CONTRIBUTING.md) for build and test instructions.
 
@@ -14,7 +14,7 @@ Read [CONTRIBUTING](CONTRIBUTING.md) for build and test instructions.
 * `ptp`: Creates a veth pair.
 * `vlan`: Allocates a vlan device.
 * `host-device`: Move an already-existing device into a container.
-#### Windows: windows specific
+#### Windows: Windows specific
 * `win-bridge`: Creates a bridge, adds the host and the container to it.
 * `win-overlay`: Creates an overlay interface to the container.
 ### IPAM: IP address allocation
@@ -23,7 +23,6 @@ Read [CONTRIBUTING](CONTRIBUTING.md) for build and test instructions.
 * `static`:  Allocate a static IPv4/IPv6 addresses to container and it's useful in debugging purpose.
 
 ### Meta: other plugins
-* `flannel`: Generates an interface corresponding to a flannel config file
 * `tuning`: Tweaks sysctl parameters of an existing interface
 * `portmap`: An iptables-based portmapping plugin. Maps ports from the host's address space to the container.
 * `bandwidth`: Allows bandwidth-limiting through use of traffic control tbf (ingress/egress).

build_linux.sh

@@ -1,21 +1,11 @@
 #!/usr/bin/env bash
 set -e
-cd $(dirname "$0")
+cd "$(dirname "$0")"
 
 if [ "$(uname)" == "Darwin" ]; then
 	export GOOS="${GOOS:-linux}"
 fi
 
-ORG_PATH="github.com/containernetworking"
-export REPO_PATH="${ORG_PATH}/plugins"
-
-if [ ! -h gopath/src/${REPO_PATH} ]; then
-	mkdir -p gopath/src/${ORG_PATH}
-	ln -s ../../../.. gopath/src/${REPO_PATH} || exit 255
-fi
-
-export GOPATH=${PWD}/gopath
-export GO="${GO:-go}"
 export GOFLAGS="${GOFLAGS} -mod=vendor"
 
 mkdir -p "${PWD}/bin"
@@ -25,9 +15,9 @@ PLUGINS="plugins/meta/* plugins/main/* plugins/ipam/*"
 for d in $PLUGINS; do
 	if [ -d "$d" ]; then
 		plugin="$(basename "$d")"
-		if [ $plugin != "windows" ]; then
+		if [ "${plugin}" != "windows" ]; then
 			echo "  $plugin"
-			$GO build -o "${PWD}/bin/$plugin" "$@" "$REPO_PATH"/$d
+			${GO:-go} build -o "${PWD}/bin/$plugin" "$@" ./"$d"
 		fi
 	fi
 done

build_windows.sh

@@ -1,25 +1,15 @@
 #!/usr/bin/env bash
 set -e
-cd $(dirname "$0")
-
-ORG_PATH="github.com/containernetworking"
-export REPO_PATH="${ORG_PATH}/plugins"
-
-export GOPATH=$(mktemp -d)
-mkdir -p ${GOPATH}/src/${ORG_PATH}
-trap "{ rm -rf $GOPATH; }" EXIT
-ln -s ${PWD} ${GOPATH}/src/${REPO_PATH} || exit 255
+cd "$(dirname "$0")"
 
 export GO="${GO:-go}"
 export GOOS=windows
 export GOFLAGS="${GOFLAGS} -mod=vendor"
-echo $GOFLAGS
+echo "$GOFLAGS"
 
-PLUGINS=$(cat plugins/windows_only.txt)
+PLUGINS=$(cat plugins/windows_only.txt | dos2unix )
 for d in $PLUGINS; do
-  if [ -d "$d" ]; then
-    plugin="$(basename "$d").exe"
-    echo "  $plugin"
-    $GO build -o "${PWD}/bin/$plugin" "$@" "$REPO_PATH"/$d
-  fi
+	plugin="$(basename "$d").exe"
+	echo "building $plugin"
+	$GO build -o "${PWD}/bin/$plugin" "$@" ./"${d}"
 done

go.mod

@@ -1,33 +1,40 @@
 module github.com/containernetworking/plugins
 
-go 1.14
+go 1.17
 
 require (
-	github.com/Microsoft/go-winio v0.4.11 // indirect
-	github.com/Microsoft/hcsshim v0.8.6
-	github.com/alexflint/go-filemutex v0.0.0-20171022225611-72bdc8eae2ae
-	github.com/buger/jsonparser v0.0.0-20180808090653-f4dd9f5a6b44
-	github.com/containernetworking/cni v0.8.0
-	github.com/coreos/go-iptables v0.4.5
-	github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7
+	github.com/Microsoft/hcsshim v0.8.20
+	github.com/alexflint/go-filemutex v1.1.0
+	github.com/buger/jsonparser v1.1.1
+	github.com/containernetworking/cni v1.0.1
+	github.com/coreos/go-iptables v0.6.0
+	github.com/coreos/go-systemd/v22 v22.3.2
 	github.com/d2g/dhcp4 v0.0.0-20170904100407-a1d1b6c41b1c
 	github.com/d2g/dhcp4client v1.0.0
 	github.com/d2g/dhcp4server v0.0.0-20181031114812-7d4a0a7f59a5
-	github.com/d2g/hardwareaddr v0.0.0-20190221164911-e7d9fbe030e4 // indirect
-	github.com/godbus/dbus v0.0.0-20180201030542-885f9cc04c9c
-	github.com/golang/protobuf v1.3.1 // indirect
-	github.com/j-keck/arping v0.0.0-20160618110441-2cf9dc699c56
-	github.com/mattn/go-shellwords v1.0.3
-	github.com/onsi/ginkgo v0.0.0-20151202141238-7f8ab55aaf3b
-	github.com/onsi/gomega v0.0.0-20151007035656-2152b45fa28a
-	github.com/safchain/ethtool v0.0.0-20190326074333-42ed695e3de8
-	github.com/sirupsen/logrus v1.0.6 // indirect
-	github.com/stretchr/testify v1.3.0 // indirect
-	github.com/vishvananda/netlink v0.0.0-20181108222139-023a6dafdcdf
-	github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc // indirect
-	golang.org/x/crypto v0.0.0-20181009213950-7c1a557ab941 // indirect
-	golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1 // indirect
-	golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f
-	gopkg.in/airbrake/gobrake.v2 v2.0.9 // indirect
-	gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2 // indirect
+	github.com/godbus/dbus/v5 v5.0.4
+	github.com/mattn/go-shellwords v1.0.12
+	github.com/networkplumbing/go-nft v0.2.0
+	github.com/onsi/ginkgo v1.16.4
+	github.com/onsi/gomega v1.15.0
+	github.com/safchain/ethtool v0.0.0-20210803160452-9aa261dae9b1
+	github.com/vishvananda/netlink v1.1.1-0.20210330154013-f5de75959ad5
+	golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e
+)
+
+require (
+	github.com/Microsoft/go-winio v0.4.17 // indirect
+	github.com/containerd/cgroups v1.0.1 // indirect
+	github.com/fsnotify/fsnotify v1.4.9 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
+	github.com/nxadm/tail v1.4.8 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/sirupsen/logrus v1.8.1 // indirect
+	github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f // indirect
+	go.opencensus.io v0.22.3 // indirect
+	golang.org/x/net v0.0.0-20210428140749-89ef3d95e781 // indirect
+	golang.org/x/text v0.3.6 // indirect
+	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
 )

go.sum

@@ -1,17 +1,213 @@
-github.com/Microsoft/go-winio v0.4.11 h1:zoIOcVf0xPN1tnMVbTtEdI+P8OofVk3NObnwOQ6nK2Q=
+bazil.org/fuse v0.0.0-20160811212531-371fbbdaa898/go.mod h1:Xbm+BRKSBEpa4q4hTSxohYNQpsxXPbPry4JJWOB3LB8=
+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
+cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=
+cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
+cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=
+cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=
+cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To=
+cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4=
+cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M=
+cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc=
+cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
+cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
+cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc=
+cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
+cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
+cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
+cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
+cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA=
+cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
+cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
+cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
+dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
+github.com/Azure/azure-sdk-for-go v16.2.1+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
+github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
+github.com/Azure/go-autorest v10.8.1+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=
+github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=
+github.com/Azure/go-autorest/autorest v0.11.1/go.mod h1:JFgpikqFJ/MleTTxwepExTKnFUKKszPS8UavbQYUMuw=
+github.com/Azure/go-autorest/autorest/adal v0.9.0/go.mod h1:/c022QCutn2P7uY+/oQWWNcK9YU+MH96NgK+jErpbcg=
+github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A=
+github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74=
+github.com/Azure/go-autorest/autorest/mocks v0.4.0/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k=
+github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k=
+github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8=
+github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA=
-github.com/Microsoft/hcsshim v0.8.6 h1:ZfF0+zZeYdzMIVMZHKtDKJvLHj76XCuVae/jNkjj0IA=
+github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA=
+github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5/go.mod h1:tTuCMEN+UleMWgg9dVx4Hu52b1bJo+59jBh3ajtinzw=
+github.com/Microsoft/go-winio v0.4.16-0.20201130162521-d1ffc52c7331/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0=
+github.com/Microsoft/go-winio v0.4.16/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0=
+github.com/Microsoft/go-winio v0.4.17-0.20210211115548-6eac466e5fa3/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=
+github.com/Microsoft/go-winio v0.4.17-0.20210324224401-5516f17a5958/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=
+github.com/Microsoft/go-winio v0.4.17 h1:iT12IBVClFevaf8PuVyi3UmZOVh4OqnaLxDTW2O6j3w=
+github.com/Microsoft/go-winio v0.4.17/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=
 github.com/Microsoft/hcsshim v0.8.6/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg=
-github.com/alexflint/go-filemutex v0.0.0-20171022225611-72bdc8eae2ae h1:AMzIhMUqU3jMrZiTuW0zkYeKlKDAFD+DG20IoO421/Y=
+github.com/Microsoft/hcsshim v0.8.7-0.20190325164909-8abdbb8205e4/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg=
+github.com/Microsoft/hcsshim v0.8.7/go.mod h1:OHd7sQqRFrYd3RmSgbgji+ctCwkbq2wbEYNSzOYtcBQ=
+github.com/Microsoft/hcsshim v0.8.9/go.mod h1:5692vkUqntj1idxauYlpoINNKeqCiG6Sg38RRsjT5y8=
+github.com/Microsoft/hcsshim v0.8.14/go.mod h1:NtVKoYxQuTLx6gEq0L96c9Ju4JbRJ4nY2ow3VK6a9Lg=
+github.com/Microsoft/hcsshim v0.8.15/go.mod h1:x38A4YbHbdxJtc0sF6oIz+RG0npwSCAvn69iY6URG00=
+github.com/Microsoft/hcsshim v0.8.16/go.mod h1:o5/SZqmR7x9JNKsW3pu+nqHm0MF8vbA+VxGOoXdC600=
+github.com/Microsoft/hcsshim v0.8.20 h1:ZTwcx3NS8n07kPf/JZ1qwU6vnjhVPMUWlXBF8r9UxrE=
+github.com/Microsoft/hcsshim v0.8.20/go.mod h1:+w2gRZ5ReXQhFOrvSQeNfhrYB/dg3oDwTOcER2fw4I4=
+github.com/Microsoft/hcsshim/test v0.0.0-20201218223536-d3e5debf77da/go.mod h1:5hlzMzRKMLyo42nCZ9oml8AdTlq/0cvIaBv6tK1RehU=
+github.com/Microsoft/hcsshim/test v0.0.0-20210227013316-43a75bb4edd3/go.mod h1:mw7qgWloBUl75W/gVH3cQszUg1+gUITj7D6NY7ywVnY=
+github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ=
+github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
+github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
+github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
+github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d/go.mod h1:HI8ITrYtUY+O+ZhtlqUnD8+KwNPOyugEhfP9fdUIaEQ=
+github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/alexflint/go-filemutex v0.0.0-20171022225611-72bdc8eae2ae/go.mod h1:CgnQgUtFrFz9mxFNtED3jI5tLDjKlOM+oUF/sTk6ps0=
-github.com/buger/jsonparser v0.0.0-20180808090653-f4dd9f5a6b44 h1:y853v6rXx+zefEcjET3JuKAqvhj+FKflQijjeaSv2iA=
+github.com/alexflint/go-filemutex v1.1.0 h1:IAWuUuRYL2hETx5b8vCgwnD+xSdlsTQY6s2JjBsqLdg=
+github.com/alexflint/go-filemutex v1.1.0/go.mod h1:7P4iRhttt/nUvUOrYIhcpMzv2G6CY9UnI16Z+UJqRyk=
+github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
+github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
+github.com/aws/aws-sdk-go v1.15.11/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0=
+github.com/beorn7/perks v0.0.0-20160804104726-4c0e84591b9a/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
+github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
+github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
+github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
+github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA=
+github.com/blang/semver v3.1.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
+github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
+github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4=
+github.com/bshuster-repo/logrus-logstash-hook v0.4.1/go.mod h1:zsTqEiSzDgAa/8GZR7E1qaXrhYNDKBYy5/dWPTIflbk=
 github.com/buger/jsonparser v0.0.0-20180808090653-f4dd9f5a6b44/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s=
-github.com/containernetworking/cni v0.8.0 h1:BT9lpgGoH4jw3lFC7Odz2prU5ruiYKcgAjMCbgybcKI=
+github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs=
+github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0=
+github.com/bugsnag/bugsnag-go v0.0.0-20141110184014-b1d153021fcd/go.mod h1:2oa8nejYd4cQ/b0hMIopN0lCRxU0bueqREvZLWFrtK8=
+github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b/go.mod h1:obH5gd0BsqsP2LwDJ9aOkm/6J86V6lyAXCoQWGw3K50=
+github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0/go.mod h1:D/8v3kj0zr8ZAKg1AQ6crr+5VwKN5eIywRkfhyM/+dE=
+github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
+github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/checkpoint-restore/go-criu/v4 v4.1.0/go.mod h1:xUQBLp4RLc5zJtWY++yjOoMoB5lihDt7fai+75m+rGw=
+github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
+github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
+github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
+github.com/cilium/ebpf v0.0.0-20200110133405-4032b1d8aae3/go.mod h1:MA5e5Lr8slmEg9bt0VpxxWqJlO4iwu3FBdHUzV7wQVg=
+github.com/cilium/ebpf v0.0.0-20200702112145-1c8d4c9ef775/go.mod h1:7cR51M8ViRLIdUjrmSXlK9pkrsDlLHbO8jiB8X8JnOc=
+github.com/cilium/ebpf v0.2.0/go.mod h1:To2CFviqOWL/M0gIMsvSMlqe7em/l1ALkX1PyjrX2Qs=
+github.com/cilium/ebpf v0.4.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
+github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8=
+github.com/containerd/aufs v0.0.0-20200908144142-dab0cbea06f4/go.mod h1:nukgQABAEopAHvB6j7cnP5zJ+/3aVcE7hCYqvIwAHyE=
+github.com/containerd/aufs v0.0.0-20201003224125-76a6863f2989/go.mod h1:AkGGQs9NM2vtYHaUen+NljV0/baGCAPELGm2q9ZXpWU=
+github.com/containerd/aufs v0.0.0-20210316121734-20793ff83c97/go.mod h1:kL5kd6KM5TzQjR79jljyi4olc1Vrx6XBlcyj3gNv2PU=
+github.com/containerd/aufs v1.0.0/go.mod h1:kL5kd6KM5TzQjR79jljyi4olc1Vrx6XBlcyj3gNv2PU=
+github.com/containerd/btrfs v0.0.0-20201111183144-404b9149801e/go.mod h1:jg2QkJcsabfHugurUvvPhS3E08Oxiuh5W/g1ybB4e0E=
+github.com/containerd/btrfs v0.0.0-20210316141732-918d888fb676/go.mod h1:zMcX3qkXTAi9GI50+0HOeuV8LU2ryCE/V2vG/ZBiTss=
+github.com/containerd/btrfs v1.0.0/go.mod h1:zMcX3qkXTAi9GI50+0HOeuV8LU2ryCE/V2vG/ZBiTss=
+github.com/containerd/cgroups v0.0.0-20190717030353-c4b9ac5c7601/go.mod h1:X9rLEHIqSf/wfK8NsPqxJmeZgW4pcfzdXITDrUSJ6uI=
+github.com/containerd/cgroups v0.0.0-20190919134610-bf292b21730f/go.mod h1:OApqhQ4XNSNC13gXIwDjhOQxjWa/NxkwZXJ1EvqT0ko=
+github.com/containerd/cgroups v0.0.0-20200531161412-0dbf7f05ba59/go.mod h1:pA0z1pT8KYB3TCXK/ocprsh7MAkoW8bZVzPdih9snmM=
+github.com/containerd/cgroups v0.0.0-20200710171044-318312a37340/go.mod h1:s5q4SojHctfxANBDvMeIaIovkq29IP48TKAxnhYRxvo=
+github.com/containerd/cgroups v0.0.0-20200824123100-0b889c03f102/go.mod h1:s5q4SojHctfxANBDvMeIaIovkq29IP48TKAxnhYRxvo=
+github.com/containerd/cgroups v0.0.0-20210114181951-8a68de567b68/go.mod h1:ZJeTFisyysqgcCdecO57Dj79RfL0LNeGiFUqLYQRYLE=
+github.com/containerd/cgroups v1.0.1 h1:iJnMvco9XGvKUvNQkv88bE4uJXxRQH18efbKo9w5vHQ=
+github.com/containerd/cgroups v1.0.1/go.mod h1:0SJrPIenamHDcZhEcJMNBB85rHcUsw4f25ZfBiPYRkU=
+github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw=
+github.com/containerd/console v0.0.0-20181022165439-0650fd9eeb50/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw=
+github.com/containerd/console v0.0.0-20191206165004-02ecf6a7291e/go.mod h1:8Pf4gM6VEbTNRIT26AyyU7hxdQU3MvAvxVI0sc00XBE=
+github.com/containerd/console v1.0.1/go.mod h1:XUsP6YE/mKtz6bxc+I8UiKKTP04qjQL4qcS3XoQ5xkw=
+github.com/containerd/console v1.0.2/go.mod h1:ytZPjGgY2oeTkAONYafi2kSj0aYggsf8acV1PGKCbzQ=
+github.com/containerd/containerd v1.2.10/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
+github.com/containerd/containerd v1.3.0-beta.2.0.20190828155532-0293cbd26c69/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
+github.com/containerd/containerd v1.3.0/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
+github.com/containerd/containerd v1.3.1-0.20191213020239-082f7e3aed57/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
+github.com/containerd/containerd v1.3.2/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
+github.com/containerd/containerd v1.4.0-beta.2.0.20200729163537-40b22ef07410/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
+github.com/containerd/containerd v1.4.1/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
+github.com/containerd/containerd v1.4.3/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
+github.com/containerd/containerd v1.5.0-beta.1/go.mod h1:5HfvG1V2FsKesEGQ17k5/T7V960Tmcumvqn8Mc+pCYQ=
+github.com/containerd/containerd v1.5.0-beta.3/go.mod h1:/wr9AVtEM7x9c+n0+stptlo/uBBoBORwEx6ardVcmKU=
+github.com/containerd/containerd v1.5.0-beta.4/go.mod h1:GmdgZd2zA2GYIBZ0w09ZvgqEq8EfBp/m3lcVZIvPHhI=
+github.com/containerd/containerd v1.5.0-rc.0/go.mod h1:V/IXoMqNGgBlabz3tHD2TWDoTJseu1FGOKuoA4nNb2s=
+github.com/containerd/containerd v1.5.1/go.mod h1:0DOxVqwDy2iZvrZp2JUx/E+hS0UNTVn7dJnIOwtYR4g=
+github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=
+github.com/containerd/continuity v0.0.0-20190815185530-f2a389ac0a02/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=
+github.com/containerd/continuity v0.0.0-20191127005431-f65d91d395eb/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=
+github.com/containerd/continuity v0.0.0-20200710164510-efbc4488d8fe/go.mod h1:cECdGN1O8G9bgKTlLhuPJimka6Xb/Gg7vYzCTNVxhvo=
+github.com/containerd/continuity v0.0.0-20201208142359-180525291bb7/go.mod h1:kR3BEg7bDFaEddKm54WSmrol1fKWDU1nKYkgrcgZT7Y=
+github.com/containerd/continuity v0.0.0-20210208174643-50096c924a4e/go.mod h1:EXlVlkqNba9rJe3j7w3Xa924itAMLgZH4UD/Q4PExuQ=
+github.com/containerd/continuity v0.1.0/go.mod h1:ICJu0PwR54nI0yPEnJ6jcS+J7CZAUXrLh8lPo2knzsM=
+github.com/containerd/fifo v0.0.0-20180307165137-3d5202aec260/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI=
+github.com/containerd/fifo v0.0.0-20190226154929-a9fb20d87448/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI=
+github.com/containerd/fifo v0.0.0-20200410184934-f15a3290365b/go.mod h1:jPQ2IAeZRCYxpS/Cm1495vGFww6ecHmMk1YJH2Q5ln0=
+github.com/containerd/fifo v0.0.0-20201026212402-0724c46b320c/go.mod h1:jPQ2IAeZRCYxpS/Cm1495vGFww6ecHmMk1YJH2Q5ln0=
+github.com/containerd/fifo v0.0.0-20210316144830-115abcc95a1d/go.mod h1:ocF/ME1SX5b1AOlWi9r677YJmCPSwwWnQ9O123vzpE4=
+github.com/containerd/fifo v1.0.0/go.mod h1:ocF/ME1SX5b1AOlWi9r677YJmCPSwwWnQ9O123vzpE4=
+github.com/containerd/go-cni v1.0.1/go.mod h1:+vUpYxKvAF72G9i1WoDOiPGRtQpqsNW/ZHtSlv++smU=
+github.com/containerd/go-cni v1.0.2/go.mod h1:nrNABBHzu0ZwCug9Ije8hL2xBCYh/pjfMb1aZGrrohk=
+github.com/containerd/go-runc v0.0.0-20180907222934-5a6d9f37cfa3/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0=
+github.com/containerd/go-runc v0.0.0-20190911050354-e029b79d8cda/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0=
+github.com/containerd/go-runc v0.0.0-20200220073739-7016d3ce2328/go.mod h1:PpyHrqVs8FTi9vpyHwPwiNEGaACDxT/N/pLcvMSRA9g=
+github.com/containerd/go-runc v0.0.0-20201020171139-16b287bc67d0/go.mod h1:cNU0ZbCgCQVZK4lgG3P+9tn9/PaJNmoDXPpoJhDR+Ok=
+github.com/containerd/go-runc v1.0.0/go.mod h1:cNU0ZbCgCQVZK4lgG3P+9tn9/PaJNmoDXPpoJhDR+Ok=
+github.com/containerd/imgcrypt v1.0.1/go.mod h1:mdd8cEPW7TPgNG4FpuP3sGBiQ7Yi/zak9TYCG3juvb0=
+github.com/containerd/imgcrypt v1.0.4-0.20210301171431-0ae5c75f59ba/go.mod h1:6TNsg0ctmizkrOgXRNQjAPFWpMYRWuiB6dSF4Pfa5SA=
+github.com/containerd/imgcrypt v1.1.1-0.20210312161619-7ed62a527887/go.mod h1:5AZJNI6sLHJljKuI9IHnw1pWqo/F0nGDOuR9zgTs7ow=
+github.com/containerd/imgcrypt v1.1.1/go.mod h1:xpLnwiQmEUJPvQoAapeb2SNCxz7Xr6PJrXQb0Dpc4ms=
+github.com/containerd/nri v0.0.0-20201007170849-eb1350a75164/go.mod h1:+2wGSDGFYfE5+So4M5syatU0N0f0LbWpuqyMi4/BE8c=
+github.com/containerd/nri v0.0.0-20210316161719-dbaa18c31c14/go.mod h1:lmxnXF6oMkbqs39FiCt1s0R2HSMhcLel9vNL3m4AaeY=
+github.com/containerd/nri v0.1.0/go.mod h1:lmxnXF6oMkbqs39FiCt1s0R2HSMhcLel9vNL3m4AaeY=
+github.com/containerd/ttrpc v0.0.0-20190828154514-0e0f228740de/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o=
+github.com/containerd/ttrpc v0.0.0-20190828172938-92c8520ef9f8/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o=
+github.com/containerd/ttrpc v0.0.0-20191028202541-4f1b8fe65a5c/go.mod h1:LPm1u0xBw8r8NOKoOdNMeVHSawSsltak+Ihv+etqsE8=
+github.com/containerd/ttrpc v1.0.1/go.mod h1:UAxOpgT9ziI0gJrmKvgcZivgxOp8iFPSk8httJEt98Y=
+github.com/containerd/ttrpc v1.0.2/go.mod h1:UAxOpgT9ziI0gJrmKvgcZivgxOp8iFPSk8httJEt98Y=
+github.com/containerd/typeurl v0.0.0-20180627222232-a93fcdb778cd/go.mod h1:Cm3kwCdlkCfMSHURc+r6fwoGH6/F1hH3S4sg0rLFWPc=
+github.com/containerd/typeurl v0.0.0-20190911142611-5eb25027c9fd/go.mod h1:GeKYzf2pQcqv7tJ0AoCuuhtnqhva5LNU3U+OyKxxJpk=
+github.com/containerd/typeurl v1.0.1/go.mod h1:TB1hUtrpaiO88KEK56ijojHS1+NeF0izUACaJW2mdXg=
+github.com/containerd/typeurl v1.0.2/go.mod h1:9trJWW2sRlGub4wZJRTW83VtbOLS6hwcDZXTn6oPz9s=
+github.com/containerd/zfs v0.0.0-20200918131355-0a33824f23a2/go.mod h1:8IgZOBdv8fAgXddBT4dBXJPtxyRsejFIpXoklgxgEjw=
+github.com/containerd/zfs v0.0.0-20210301145711-11e8f1707f62/go.mod h1:A9zfAbMlQwE+/is6hi0Xw8ktpL+6glmqZYtevJgaB8Y=
+github.com/containerd/zfs v0.0.0-20210315114300-dde8f0fda960/go.mod h1:m+m51S1DvAP6r3FcmYCp54bQ34pyOwTieQDNRIRHsFY=
+github.com/containerd/zfs v0.0.0-20210324211415-d5c4544f0433/go.mod h1:m+m51S1DvAP6r3FcmYCp54bQ34pyOwTieQDNRIRHsFY=
+github.com/containerd/zfs v1.0.0/go.mod h1:m+m51S1DvAP6r3FcmYCp54bQ34pyOwTieQDNRIRHsFY=
+github.com/containernetworking/cni v0.7.1/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY=
 github.com/containernetworking/cni v0.8.0/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY=
-github.com/coreos/go-iptables v0.4.5 h1:DpHb9vJrZQEFMcVLFKAAGMUVX0XoRC0ptCthinRYm38=
+github.com/containernetworking/cni v0.8.1/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY=
+github.com/containernetworking/cni v1.0.1 h1:9OIL/sZmMYDBe+G8svzILAlulUpaDTUjeAbtH/JNLBo=
+github.com/containernetworking/cni v1.0.1/go.mod h1:AKuhXbN5EzmD4yTNtfSsX3tPcmtrBI6QcRV0NiNt15Y=
+github.com/containernetworking/plugins v0.8.6/go.mod h1:qnw5mN19D8fIwkqW7oHHYDHVlzhJpcY6TQxn/fUyDDM=
+github.com/containernetworking/plugins v0.9.1/go.mod h1:xP/idU2ldlzN6m4p5LmGiwRDjeJr6FLK6vuiUwoH7P8=
+github.com/containers/ocicrypt v1.0.1/go.mod h1:MeJDzk1RJHv89LjsH0Sp5KTY3ZYkjXO/C+bKAeWFIrc=
+github.com/containers/ocicrypt v1.1.0/go.mod h1:b8AOe0YR67uU8OqfVNcznfFpAzu3rdgUV4GP9qXPfu4=
+github.com/containers/ocicrypt v1.1.1/go.mod h1:Dm55fwWm1YZAjYRaJ94z2mfZikIyIN4B0oB3dj3jFxY=
+github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
+github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/go-iptables v0.4.5/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU=
-github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7 h1:u9SHYsPQNyt5tgDm3YN7+9dYrpK96E5wFilTFWIDZOM=
+github.com/coreos/go-iptables v0.5.0/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU=
+github.com/coreos/go-iptables v0.6.0 h1:is9qnZMPYjLd8LYqmm/qlE+wwEgJIkTYdhV3rfZo4jk=
+github.com/coreos/go-iptables v0.6.0/go.mod h1:Qe8Bv2Xik5FyTXwgIbLAnv2sWSBmvWdFETJConOQ//Q=
+github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc=
+github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
+github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
+github.com/coreos/go-systemd v0.0.0-20161114122254-48702e0da86b/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
 github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
+github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e h1:Wf6HqHfScWJN9/ZjdUKyjop4mf3Qdd+1TvvltAvM3m8=
+github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
+github.com/coreos/go-systemd/v22 v22.0.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk=
+github.com/coreos/go-systemd/v22 v22.1.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk=
+github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzAJc1DzSI=
+github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
+github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
+github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
+github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
+github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
+github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
+github.com/cyphar/filepath-securejoin v0.2.2/go.mod h1:FpkQEhXnPnOthhzymB7CGsFk2G9VLXONKD9G7QGMM+4=
 github.com/d2g/dhcp4 v0.0.0-20170904100407-a1d1b6c41b1c h1:Xo2rK1pzOm0jO6abTPIQwbAmqBIOj132otexc1mmzFc=
 github.com/d2g/dhcp4 v0.0.0-20170904100407-a1d1b6c41b1c/go.mod h1:Ct2BUK8SB0YC1SMSibvLzxjeJLnrYEVLULFNiHY9YfQ=
 github.com/d2g/dhcp4client v1.0.0 h1:suYBsYZIkSlUMEz4TAYCczKf62IA2UWC+O8+KtdOhCo=
@@ -20,40 +216,762 @@ github.com/d2g/dhcp4server v0.0.0-20181031114812-7d4a0a7f59a5 h1:+CpLbZIeUn94m02
 github.com/d2g/dhcp4server v0.0.0-20181031114812-7d4a0a7f59a5/go.mod h1:Eo87+Kg/IX2hfWJfwxMzLyuSZyxSoAug2nGa1G2QAi8=
 github.com/d2g/hardwareaddr v0.0.0-20190221164911-e7d9fbe030e4 h1:itqmmf1PFpC4n5JW+j4BU7X4MTfVurhYRTjODoPb2Y8=
 github.com/d2g/hardwareaddr v0.0.0-20190221164911-e7d9fbe030e4/go.mod h1:bMl4RjIciD2oAxI7DmWRx6gbeqrkoLqv3MV0vzNad+I=
-github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/godbus/dbus v0.0.0-20180201030542-885f9cc04c9c h1:RBUpb2b14UnmRHNd2uHz20ZHLDK+SW5Us/vWF5IHRaY=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/denverdino/aliyungo v0.0.0-20190125010748-a747050bb1ba/go.mod h1:dV8lFg6daOBZbT6/BDGIz6Y3WFGn8juu6G+CQ6LHtl0=
+github.com/dgrijalva/jwt-go v0.0.0-20170104182250-a601269ab70c/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
+github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
+github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
+github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E=
+github.com/docker/distribution v0.0.0-20190905152932-14b96e55d84c/go.mod h1:0+TTO4EOBfRPhZXAeF1Vu+W3hHZ8eLp8PgKVZlcvtFY=
+github.com/docker/distribution v2.7.1-0.20190205005809-0d3efadf0154+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
+github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
+github.com/docker/go-events v0.0.0-20170721190031-9461782956ad/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA=
+github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA=
+github.com/docker/go-metrics v0.0.0-20180209012529-399ea8c73916/go.mod h1:/u0gXw0Gay3ceNrsHubL3BtdOL2fHf93USgMTe0W5dI=
+github.com/docker/go-metrics v0.0.1/go.mod h1:cG1hvH2utMXtqgqqYE9plW6lDxS3/5ayHzueweSI3Vw=
+github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
+github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE=
+github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM=
+github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
+github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
+github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
+github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
+github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
+github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
+github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
+github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
+github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
+github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
+github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k=
+github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
+github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
+github.com/fullsailor/pkcs7 v0.0.0-20190404230743-d7302db945fa/go.mod h1:KnogPXtdwXqoenmZCw6S+25EAm2MkxbG0deNDu4cbSA=
+github.com/garyburd/redigo v0.0.0-20150301180006-535138d7bcd7/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY=
+github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
+github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
+github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
+github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
+github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
+github.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
+github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
+github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
+github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
+github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
+github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas=
+github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
+github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg=
+github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
+github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc=
+github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8=
+github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo=
+github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
+github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
+github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
+github.com/godbus/dbus v0.0.0-20151105175453-c7fdd8b5cd55/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw=
 github.com/godbus/dbus v0.0.0-20180201030542-885f9cc04c9c/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw=
-github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg=
+github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e h1:BWhy2j3IXJhjCbC68FptL43tDKIq8FladmaTs3Xs7Z8=
+github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4=
+github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
+github.com/godbus/dbus/v5 v5.0.4 h1:9349emZab16e7zQvpmsbtjc18ykshndd8y2PG3sgJbA=
+github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
+github.com/gogo/googleapis v1.2.0/go.mod h1:Njal3psf3qN6dwBtQfUmBZh2ybovJ0tlu3o/AC7HYjU=
+github.com/gogo/googleapis v1.4.0/go.mod h1:5YRNX2z1oM5gXdAkurHa942MDgEJyk02w4OecKY87+c=
+github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
+github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
+github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
+github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
+github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
+github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
+github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY=
+github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
+github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
+github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/j-keck/arping v0.0.0-20160618110441-2cf9dc699c56 h1:742eGXur0715JMq73aD95/FU0XpVKXqNuTnEfXsLOYQ=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
+github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
+github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
+github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
+github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
+github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
+github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
+github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
+github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg=
+github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
+github.com/gorilla/handlers v0.0.0-20150720190736-60c7bfde3e33/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ=
+github.com/gorilla/mux v1.7.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
+github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
+github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
+github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
+github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
+github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
+github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
+github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
+github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
+github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
+github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
+github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
+github.com/hashicorp/go-multierror v0.0.0-20161216184304-ed905158d874/go.mod h1:JMRHfdO9jKNzS/+BTlxCjKNQHg/jZAft8U7LloJvN7I=
+github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
+github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
+github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
+github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
+github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
+github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
+github.com/imdario/mergo v0.3.10/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
+github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
+github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/j-keck/arping v0.0.0-20160618110441-2cf9dc699c56/go.mod h1:ymszkNOg6tORTn+6F6j+Jc8TOr5osrynvN6ivFWZ2GA=
-github.com/mattn/go-shellwords v1.0.3 h1:K/VxK7SZ+cvuPgFSLKi5QPI9Vr/ipOf4C1gN+ntueUk=
+github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
+github.com/jmespath/go-jmespath v0.0.0-20160803190731-bd40a432e4c7/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
+github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
+github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
+github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
+github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
+github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
+github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
+github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
+github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
+github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
+github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/klauspost/compress v1.11.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
+github.com/klauspost/compress v1.11.13/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
+github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
+github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
+github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA=
+github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
+github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
+github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
+github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs=
+github.com/marstr/guid v1.1.0/go.mod h1:74gB1z2wpxxInTG6yaqA7KrtM0NZ+RbrcqDvYHefzho=
+github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
+github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
+github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
 github.com/mattn/go-shellwords v1.0.3/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o=
-github.com/onsi/ginkgo v0.0.0-20151202141238-7f8ab55aaf3b h1:Ey6yH0acn50T/v6CB75bGP4EMJqnv9WvnjN7oZaj+xE=
+github.com/mattn/go-shellwords v1.0.12 h1:M2zGm7EW6UQJvDeQxo4T51eKPurbeFbe8WtebGE2xrk=
+github.com/mattn/go-shellwords v1.0.12/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y=
+github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
+github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
+github.com/miekg/pkcs11 v1.0.3/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs=
+github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4=
+github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/mitchellh/osext v0.0.0-20151018003038-5e2d6d41470f/go.mod h1:OkQIRizQZAeMln+1tSwduZz7+Af5oFlKirV/MSYes2A=
+github.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc=
+github.com/moby/sys/mountinfo v0.4.0/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A=
+github.com/moby/sys/mountinfo v0.4.1/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A=
+github.com/moby/sys/symlink v0.1.0/go.mod h1:GGDODQmbFOjFsXvfLVn3+ZRxkch54RkSiGqsZeMYowQ=
+github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ=
+github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
+github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
+github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
+github.com/ncw/swift v1.0.47/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM=
+github.com/networkplumbing/go-nft v0.2.0 h1:eKapmyVUt/3VGfhYaDos5yeprm+LPt881UeksmKKZHY=
+github.com/networkplumbing/go-nft v0.2.0/go.mod h1:HnnM+tYvlGAsMU7yoYwXEVLLiDW9gdMmb5HoGcwpuQs=
+github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
+github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
+github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
+github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
+github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
 github.com/onsi/ginkgo v0.0.0-20151202141238-7f8ab55aaf3b/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/gomega v0.0.0-20151007035656-2152b45fa28a h1:KfNOeFvoAssuZLT7IntKZElKwi/5LRuxY71k+t6rfaM=
+github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
+github.com/onsi/ginkgo v1.13.0/go.mod h1:+REjRxOmWfHCjfv9TTWB1jD1Frx4XydAD3zm1lskyM0=
+github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc=
+github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0=
 github.com/onsi/gomega v0.0.0-20151007035656-2152b45fa28a/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
+github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
+github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
+github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
+github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc=
+github.com/onsi/gomega v1.15.0 h1:WjP/FQ/sk43MRmnEcT+MlDw2TFvkrXlprrPST/IudjU=
+github.com/onsi/gomega v1.15.0/go.mod h1:cIuvLEne0aoVhAgh/O6ac0Op8WWw9H6eYCriF+tEHG0=
+github.com/opencontainers/go-digest v0.0.0-20170106003457-a6d0ee40d420/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
+github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
+github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
+github.com/opencontainers/go-digest v1.0.0-rc1.0.20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
+github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
+github.com/opencontainers/image-spec v1.0.0/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
+github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
+github.com/opencontainers/runc v0.0.0-20190115041553-12f6a991201f/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U=
+github.com/opencontainers/runc v0.1.1/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U=
+github.com/opencontainers/runc v1.0.0-rc8.0.20190926000215-3e425f80a8c9/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U=
+github.com/opencontainers/runc v1.0.0-rc9/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U=
+github.com/opencontainers/runc v1.0.0-rc93/go.mod h1:3NOsor4w32B2tC0Zbl8Knk4Wg84SM2ImC1fxBuqJ/H0=
+github.com/opencontainers/runtime-spec v0.1.2-0.20190507144316-5b71a03e2700/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
+github.com/opencontainers/runtime-spec v1.0.1/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
+github.com/opencontainers/runtime-spec v1.0.2-0.20190207185410-29686dbc5559/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
+github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
+github.com/opencontainers/runtime-spec v1.0.3-0.20200929063507-e6143ca7d51d/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
+github.com/opencontainers/runtime-tools v0.0.0-20181011054405-1d69bd0f9c39/go.mod h1:r3f7wjNzSs2extwzU3Y+6pKfobzPh+kKFJ3ofN+3nfs=
+github.com/opencontainers/selinux v1.6.0/go.mod h1:VVGKuOLlE7v4PJyT6h7mNWvq1rzqiriPsEqVhc+svHE=
+github.com/opencontainers/selinux v1.8.0/go.mod h1:RScLhm78qiWa2gbVCcGkC7tCGdgk3ogry1nUQF8Evvo=
+github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
+github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc=
+github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
+github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.8.1-0.20171018195549-f15c970de5b7/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/safchain/ethtool v0.0.0-20190326074333-42ed695e3de8 h1:2c1EFnZHIPCW8qKWgHMH/fX2PkSabFc5mrVzfUNdg5U=
+github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA=
+github.com/prometheus/client_golang v0.0.0-20180209125602-c332b6f63c06/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
+github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
+github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
+github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
+github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g=
+github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
+github.com/prometheus/client_model v0.0.0-20171117100541-99fa1f4be8e5/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
+github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
+github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/common v0.0.0-20180110214958-89604d197083/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
+github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
+github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
+github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
+github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc=
+github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
+github.com/prometheus/procfs v0.0.0-20180125133057-cb4147076ac7/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
+github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
+github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
+github.com/prometheus/procfs v0.0.0-20190522114515-bc1a522cf7b1/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
+github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
+github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ=
+github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ=
+github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A=
+github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
+github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
+github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
+github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
+github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
+github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/safchain/ethtool v0.0.0-20190326074333-42ed695e3de8/go.mod h1:Z0q5wiBQGYcxhMZ6gUqHn6pYNLypFAvaL3UvgZLR0U4=
-github.com/sirupsen/logrus v1.0.6 h1:hcP1GmhGigz/O7h1WVUM5KklBp1JoNS9FggWKdj/j3s=
+github.com/safchain/ethtool v0.0.0-20210803160452-9aa261dae9b1 h1:ZFfeKAhIQiiOrQaI3/znw0gOmYpO28Tcu1YaqMa/jtQ=
+github.com/safchain/ethtool v0.0.0-20210803160452-9aa261dae9b1/go.mod h1:Z0q5wiBQGYcxhMZ6gUqHn6pYNLypFAvaL3UvgZLR0U4=
+github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
+github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw=
+github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo=
+github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
+github.com/sirupsen/logrus v1.0.4-0.20170822132746-89742aefa4b2/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc=
 github.com/sirupsen/logrus v1.0.6/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc=
+github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
+github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
+github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
+github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
+github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
+github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
+github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
+github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
+github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
+github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
+github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
+github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
+github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
+github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
+github.com/spf13/cobra v0.0.2-0.20171109065643-2da4a54c5cee/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
+github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
+github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE=
+github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
+github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/spf13/pflag v1.0.1-0.20171106142849-4c012f6dcd95/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE=
+github.com/stefanberger/go-pkcs11uri v0.0.0-20201008174630-78d3cae3a980/go.mod h1:AO3tvPzVZ/ayst6UlUKUv6rcPQInYe3IknH3jYhAKu8=
+github.com/stretchr/objx v0.0.0-20180129172003-8a3f7159479f/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
+github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
+github.com/stretchr/testify v0.0.0-20180303142811-b89eecf5ca5d/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
-github.com/vishvananda/netlink v0.0.0-20181108222139-023a6dafdcdf h1:3J37+NPjNyGW/dbfXtj3yWuF9OEepIdGOXRaJGbORV8=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
+github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
+github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
+github.com/tchap/go-patricia v2.2.6+incompatible/go.mod h1:bmLyhP68RS6kStMGxByiQ23RP/odRBOTVjwp2cDyi6I=
+github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
+github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
+github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
+github.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
+github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
+github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
+github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
 github.com/vishvananda/netlink v0.0.0-20181108222139-023a6dafdcdf/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk=
-github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc h1:R83G5ikgLMxrBvLh22JhdfI8K6YXEPHx5P03Uu3DRs4=
+github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE=
+github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho=
+github.com/vishvananda/netlink v1.1.1-0.20210330154013-f5de75959ad5 h1:+UB2BJA852UkGH42H+Oee69djmxS3ANzl2b/JtT1YiA=
+github.com/vishvananda/netlink v1.1.1-0.20210330154013-f5de75959ad5/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho=
 github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc/go.mod h1:ZjcWmFBXmLKZu9Nxj3WKYEafiSqer2rnvPr0en9UNpI=
-golang.org/x/crypto v0.0.0-20181009213950-7c1a557ab941 h1:qBTHLajHecfu+xzRI9PqVDcqx7SdHj9d4B+EzSn3tAc=
+github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU=
+github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0=
+github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f h1:p4VB7kIXpOQvVn1ZaTIVp+3vuYAXFe3OJEvjbUYJLaA=
+github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0=
+github.com/willf/bitset v1.1.11-0.20200630133818-d5bec3311243/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4=
+github.com/willf/bitset v1.1.11/go.mod h1:83CECat5yLh5zVOf4P1ErAgKA5UDvKtgyUABdr3+MjI=
+github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
+github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ=
+github.com/xeipuuv/gojsonschema v0.0.0-20180618132009-1d523034197f/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs=
+github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
+github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
+github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
+github.com/yvasiyarov/go-metrics v0.0.0-20140926110328-57bccd1ccd43/go.mod h1:aX5oPXxHm3bOH+xeAttToC8pqch2ScQN/JoXYupl6xs=
+github.com/yvasiyarov/gorelic v0.0.0-20141212073537-a9bba5b9ab50/go.mod h1:NUSPSUX/bi6SeDMUh6brw0nXpxHnc96TguQh0+r/ssA=
+github.com/yvasiyarov/newrelic_platform_go v0.0.0-20140908184405-b21fdbd4370f/go.mod h1:GlGEuHIJweS1mbCqG+7vt2nvWLzLLnRHbXz5JKd/Qbg=
+go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
+go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
+go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
+go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489/go.mod h1:yVHk9ub3CSBatqGNg7GRmsnfLWtoW60w4eDYfh7vHDg=
+go.mozilla.org/pkcs7 v0.0.0-20200128120323-432b2356ecb1/go.mod h1:SNgMg+EgDFwmvSmLRTNKC5fegJjB7v23qTQ0XLGUNHk=
+go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
+go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
+go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+go.opencensus.io v0.22.3 h1:8sGtKOrtQqkN1bp2AtX+misvLIlOmsEsNd+9NIcPEm8=
+go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
+go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
+go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
+go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
+golang.org/x/crypto v0.0.0-20171113213409-9f005a07e0d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20181009213950-7c1a557ab941/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
-golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1 h1:Y/KGZSOdz/2r0WJ9Mkmz6NJBusp0kiNx1Cn82lzJQ6w=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
+golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=
+golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY=
+golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
+golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
+golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
+golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
+golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
+golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
+golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
+golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
+golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f h1:25KHgbfyiSm6vwQLbM3zZIe1v9p/3ea4Rz+nnM5K/i4=
+golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
+golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
+golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190619014844-b5b0513f8c1b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
+golang.org/x/net v0.0.0-20210428140749-89ef3d95e781 h1:DzZ89McO9/gWPsQXS/FVKAlG02ZjaQ6AlZRBimEYOd0=
+golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190514135907-3a4b5fb9f71f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190522044717-8097e1b27ff5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190602015325-4c4f7f33c9ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-gopkg.in/airbrake/gobrake.v2 v2.0.9 h1:7z2uVWwn7oVeeugY1DtlPAy5H+KYgB1KeKTnqjNatLo=
+golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190812073006-9eafafc0a87e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191115151921-52ab43148777/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191210023423-ac6580df4449/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200120151820-655fe14d7479/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200728102440-3e129f6d46b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200817155316-9781c653f443/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200916030750-2334cc1a136f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200922070232-aee5d888a860/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201117170446-d9b008d0a637/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201202213521-69691e467435/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e h1:WUoyKPm6nCo1BnNUvPGnFG3T5DUVem42yDJZZ4CNxMA=
+golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
+golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/api v0.0.0-20160322025152-9bf6e6e569ff/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
+google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
+google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
+google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
+google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
+google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
+google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/cloud v0.0.0-20151119220103-975617b05ea8/go.mod h1:0H1ncTHf11KCFhTc/+EFRbzSCOZx+VUbRMk55Yv5MYk=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190522204451-c2c4e71fbf69/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s=
+google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
+google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200117163144-32f20d992d24/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA=
+google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
+google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/grpc v0.0.0-20160317175043-d3ddb4469d5a/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
+google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
+google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
+google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
+google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA=
+google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
+google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
+google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
+google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
+google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
+google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk=
+google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U=
-gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2 h1:OAj3g0cR6Dx/R07QgQe8wkA9RNjB2u4i700xBkIT4e0=
+gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20141024133853-64131543e789/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw=
+gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
+gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
 gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo=
+gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
+gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
+gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
+gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=
+gopkg.in/square/go-jose.v2 v2.3.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=
+gopkg.in/square/go-jose.v2 v2.5.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
+gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
+gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
+gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk=
+gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
+honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
+k8s.io/api v0.20.1/go.mod h1:KqwcCVogGxQY3nBlRpwt+wpAMF/KjaCc7RpywacvqUo=
+k8s.io/api v0.20.4/go.mod h1:++lNL1AJMkDymriNniQsWRkMDzRaX2Y/POTUi8yvqYQ=
+k8s.io/api v0.20.6/go.mod h1:X9e8Qag6JV/bL5G6bU8sdVRltWKmdHsFUGS3eVndqE8=
+k8s.io/apimachinery v0.20.1/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU=
+k8s.io/apimachinery v0.20.4/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU=
+k8s.io/apimachinery v0.20.6/go.mod h1:ejZXtW1Ra6V1O5H8xPBGz+T3+4gfkTCeExAHKU57MAc=
+k8s.io/apiserver v0.20.1/go.mod h1:ro5QHeQkgMS7ZGpvf4tSMx6bBOgPfE+f52KwvXfScaU=
+k8s.io/apiserver v0.20.4/go.mod h1:Mc80thBKOyy7tbvFtB4kJv1kbdD0eIH8k8vianJcbFM=
+k8s.io/apiserver v0.20.6/go.mod h1:QIJXNt6i6JB+0YQRNcS0hdRHJlMhflFmsBDeSgT1r8Q=
+k8s.io/client-go v0.20.1/go.mod h1:/zcHdt1TeWSd5HoUe6elJmHSQ6uLLgp4bIJHVEuy+/Y=
+k8s.io/client-go v0.20.4/go.mod h1:LiMv25ND1gLUdBeYxBIwKpkSC5IsozMMmOOeSJboP+k=
+k8s.io/client-go v0.20.6/go.mod h1:nNQMnOvEUEsOzRRFIIkdmYOjAZrC8bgq0ExboWSU1I0=
+k8s.io/component-base v0.20.1/go.mod h1:guxkoJnNoh8LNrbtiQOlyp2Y2XFCZQmrcg2n/DeYNLk=
+k8s.io/component-base v0.20.4/go.mod h1:t4p9EdiagbVCJKrQ1RsA5/V4rFQNDfRlevJajlGwgjI=
+k8s.io/component-base v0.20.6/go.mod h1:6f1MPBAeI+mvuts3sIdtpjljHWBQ2cIy38oBIWMYnrM=
+k8s.io/cri-api v0.17.3/go.mod h1:X1sbHmuXhwaHs9xxYffLqJogVsnI+f6cPRcgPel7ywM=
+k8s.io/cri-api v0.20.1/go.mod h1:2JRbKt+BFLTjtrILYVqQK5jqhI+XNdF6UiGMgczeBCI=
+k8s.io/cri-api v0.20.4/go.mod h1:2JRbKt+BFLTjtrILYVqQK5jqhI+XNdF6UiGMgczeBCI=
+k8s.io/cri-api v0.20.6/go.mod h1:ew44AjNXwyn1s0U4xCKGodU7J1HzBeZ1MpGrpa5r8Yc=
+k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
+k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE=
+k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y=
+k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM=
+k8s.io/kubernetes v1.13.0/go.mod h1:ocZa8+6APFNC2tX1DZASIbocyYT5jHzqFVsY5aoB7Jk=
+k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
+rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
+rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
+rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
+sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.14/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg=
+sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.15/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg=
+sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw=
+sigs.k8s.io/structured-merge-diff/v4 v4.0.3/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw=
+sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
+sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=

integration/integration_linux_test.go

@@ -225,23 +225,22 @@ func (n Namespace) Del() {
 }
 
 func makeTcpClientInNS(netns string, address string, port int, numBytes int) {
-	message := bytes.Repeat([]byte{'a'}, numBytes)
+	payload := bytes.Repeat([]byte{'a'}, numBytes)
+	message := string(payload)
 
-	bin, err := exec.LookPath("nc")
-	Expect(err).NotTo(HaveOccurred())
 	var cmd *exec.Cmd
 	if netns != "" {
 		netns = filepath.Base(netns)
-		cmd = exec.Command("ip", "netns", "exec", netns, bin, "-v", address, strconv.Itoa(port))
+		cmd = exec.Command("ip", "netns", "exec", netns, echoClientBinaryPath, "--target", fmt.Sprintf("%s:%d", address, port), "--message", message)
 	} else {
-		cmd = exec.Command("nc", address, strconv.Itoa(port))
+		cmd = exec.Command(echoClientBinaryPath, "--target", fmt.Sprintf("%s:%d", address, port), "--message", message)
 	}
 	cmd.Stdin = bytes.NewBuffer([]byte(message))
 	cmd.Stderr = GinkgoWriter
 	out, err := cmd.Output()
 
 	Expect(err).NotTo(HaveOccurred())
-	Expect(string(out)).To(Equal(string(message)))
+	Expect(string(out)).To(Equal(message))
 }
 
 func startEchoServerInNamespace(netNS Namespace) (int, *gexec.Session, error) {

integration/integration_suite_test.go

@@ -14,11 +14,10 @@
 package integration_test
 
 import (
-	"math/rand"
+	"strings"
 	"testing"
 
 	. "github.com/onsi/ginkgo"
-	"github.com/onsi/ginkgo/config"
 	. "github.com/onsi/gomega"
 	"github.com/onsi/gomega/gexec"
 )
@@ -28,15 +27,18 @@ func TestIntegration(t *testing.T) {
 	RunSpecs(t, "integration")
 }
 
-var echoServerBinaryPath string
+var echoServerBinaryPath, echoClientBinaryPath string
 
 var _ = SynchronizedBeforeSuite(func() []byte {
-	binaryPath, err := gexec.Build("github.com/containernetworking/plugins/pkg/testutils/echosvr")
+	serverBinaryPath, err := gexec.Build("github.com/containernetworking/plugins/pkg/testutils/echo/server")
 	Expect(err).NotTo(HaveOccurred())
-	return []byte(binaryPath)
+	clientBinaryPath, err := gexec.Build("github.com/containernetworking/plugins/pkg/testutils/echo/client")
+	Expect(err).NotTo(HaveOccurred())
+	return []byte(strings.Join([]string{serverBinaryPath, clientBinaryPath}, ","))
 }, func(data []byte) {
-	echoServerBinaryPath = string(data)
-	rand.Seed(config.GinkgoConfig.RandomSeed + int64(GinkgoParallelNode()))
+	binaries := strings.Split(string(data), ",")
+	echoServerBinaryPath = binaries[0]
+	echoClientBinaryPath = binaries[1]
 })
 
 var _ = SynchronizedAfterSuite(func() {}, func() {

pkg/hns/endpoint_windows.go

@@ -23,7 +23,8 @@ import (
 	"github.com/Microsoft/hcsshim/hcn"
 
 	"github.com/containernetworking/cni/pkg/types"
-	"github.com/containernetworking/cni/pkg/types/current"
+	current "github.com/containernetworking/cni/pkg/types/100"
+
 	"github.com/containernetworking/plugins/pkg/errors"
 )
 
@@ -40,7 +41,7 @@ type EndpointInfo struct {
 	IpAddress    net.IP
 }
 
-// GetSandboxContainerID returns the sandbox ID of this pod
+// GetSandboxContainerID returns the sandbox ID of this pod.
 func GetSandboxContainerID(containerID string, netNs string) string {
 	if len(netNs) != 0 && netNs != pauseContainerNetNS {
 		splits := strings.SplitN(netNs, ":", 2)
@@ -52,7 +53,7 @@ func GetSandboxContainerID(containerID string, netNs string) string {
 	return containerID
 }
 
-// short function so we know when to return "" for a string
+// GetIpString returns the given IP as a string.
 func GetIpString(ip *net.IP) string {
 	if len(*ip) == 0 {
 		return ""
@@ -61,222 +62,136 @@ func GetIpString(ip *net.IP) string {
 	}
 }
 
-func GenerateHnsEndpoint(epInfo *EndpointInfo, n *NetConf) (*hcsshim.HNSEndpoint, error) {
-	// run the IPAM plugin and get back the config to apply
-	hnsEndpoint, err := hcsshim.GetHNSEndpointByName(epInfo.EndpointName)
-	if err != nil && !hcsshim.IsNotExist(err) {
-		return nil, errors.Annotatef(err, "failed to get endpoint %q", epInfo.EndpointName)
+// GetDefaultDestinationPrefix returns the default destination prefix according to the given IP type.
+func GetDefaultDestinationPrefix(ip *net.IP) string {
+	destinationPrefix := "0.0.0.0/0"
+	if ip.To4() == nil {
+		destinationPrefix = "::/0"
 	}
-
-	if hnsEndpoint != nil {
-		if hnsEndpoint.VirtualNetwork != epInfo.NetworkId {
-			_, err = hnsEndpoint.Delete()
-			if err != nil {
-				return nil, errors.Annotatef(err, "failed to delete endpoint %s", epInfo.EndpointName)
-			}
-			hnsEndpoint = nil
-		}
-	}
-
-	if n.LoopbackDSR {
-		n.ApplyLoopbackDSR(&epInfo.IpAddress)
-	}
-	if hnsEndpoint == nil {
-		hnsEndpoint = &hcsshim.HNSEndpoint{
-			Name:           epInfo.EndpointName,
-			VirtualNetwork: epInfo.NetworkId,
-			DNSServerList:  strings.Join(epInfo.DNS.Nameservers, ","),
-			DNSSuffix:      strings.Join(epInfo.DNS.Search, ","),
-			GatewayAddress: GetIpString(&epInfo.Gateway),
-			IPAddress:      epInfo.IpAddress,
-			Policies:       n.MarshalPolicies(),
-		}
-	}
-	return hnsEndpoint, nil
+	return destinationPrefix
 }
 
-func GenerateHcnEndpoint(epInfo *EndpointInfo, n *NetConf) (*hcn.HostComputeEndpoint, error) {
-	// run the IPAM plugin and get back the config to apply
-	hcnEndpoint, err := hcn.GetEndpointByName(epInfo.EndpointName)
-	if err != nil && !hcn.IsNotFoundError(err) {
-		return nil, errors.Annotatef(err, "failed to get endpoint %q", epInfo.EndpointName)
-	}
-
-	if hcnEndpoint != nil {
-		// If the endpont already exists, then we should return error unless
-		// the endpoint is based on a different network then delete
-		// should that fail return error
-		if !strings.EqualFold(hcnEndpoint.HostComputeNetwork, epInfo.NetworkId) {
-			err = hcnEndpoint.Delete()
-			if err != nil {
-				return nil, errors.Annotatef(err, "failed to delete endpoint %s", epInfo.EndpointName)
-			}
-		} else {
-			return nil, fmt.Errorf("endpoint %q already exits", epInfo.EndpointName)
-		}
-	}
-
-	if hcnEndpoint == nil {
-		routes := []hcn.Route{
-			{
-				NextHop:           GetIpString(&epInfo.Gateway),
-				DestinationPrefix: GetDefaultDestinationPrefix(&epInfo.Gateway),
-			},
-		}
-
-		hcnDns := hcn.Dns{
-			Search:     epInfo.DNS.Search,
-			ServerList: epInfo.DNS.Nameservers,
-		}
-
-		hcnIpConfig := hcn.IpConfig{
-			IpAddress: GetIpString(&epInfo.IpAddress),
-		}
-		ipConfigs := []hcn.IpConfig{hcnIpConfig}
-
-		if n.LoopbackDSR {
-			n.ApplyLoopbackDSR(&epInfo.IpAddress)
-		}
-		hcnEndpoint = &hcn.HostComputeEndpoint{
-			SchemaVersion:      hcn.Version{Major: 2},
-			Name:               epInfo.EndpointName,
-			HostComputeNetwork: epInfo.NetworkId,
-			Dns:                hcnDns,
-			Routes:             routes,
-			IpConfigurations:   ipConfigs,
-			Policies: func() []hcn.EndpointPolicy {
-				if n.HcnPolicyArgs == nil {
-					n.HcnPolicyArgs = []hcn.EndpointPolicy{}
-				}
-				return n.HcnPolicyArgs
-			}(),
-		}
-	}
-	return hcnEndpoint, nil
-}
-
-// ConstructEndpointName constructs enpointId which is used to identify an endpoint from HNS
-// There is a special consideration for netNs name here, which is required for Windows Server 1709
-// containerID is the Id of the container on which the endpoint is worked on
+// ConstructEndpointName constructs endpoint id which is used to identify an endpoint from HNS/HCN.
 func ConstructEndpointName(containerID string, netNs string, networkName string) string {
 	return GetSandboxContainerID(containerID, netNs) + "_" + networkName
 }
 
-// DeprovisionEndpoint removes an endpoint from the container by sending a Detach request to HNS
-// For shared endpoint, ContainerDetach is used
-// for removing the endpoint completely, HotDetachEndpoint is used
-func DeprovisionEndpoint(epName string, netns string, containerID string) error {
+// GenerateHnsEndpoint generates an HNSEndpoint with given info and config.
+func GenerateHnsEndpoint(epInfo *EndpointInfo, n *NetConf) (*hcsshim.HNSEndpoint, error) {
+	// run the IPAM plugin and get back the config to apply
+	hnsEndpoint, err := hcsshim.GetHNSEndpointByName(epInfo.EndpointName)
+	if err != nil && !hcsshim.IsNotExist(err) {
+		return nil, errors.Annotatef(err, "failed to get HNSEndpoint %s", epInfo.EndpointName)
+	}
+
+	if hnsEndpoint != nil {
+		if strings.EqualFold(hnsEndpoint.VirtualNetwork, epInfo.NetworkId) {
+			return nil, fmt.Errorf("HNSEndpoint %s is already existed", epInfo.EndpointName)
+		}
+		// remove endpoint if corrupted
+		if _, err = hnsEndpoint.Delete(); err != nil {
+			return nil, errors.Annotatef(err, "failed to delete corrupted HNSEndpoint %s", epInfo.EndpointName)
+		}
+	}
+
+	if n.LoopbackDSR {
+		n.ApplyLoopbackDSRPolicy(&epInfo.IpAddress)
+	}
+	hnsEndpoint = &hcsshim.HNSEndpoint{
+		Name:           epInfo.EndpointName,
+		VirtualNetwork: epInfo.NetworkId,
+		DNSServerList:  strings.Join(epInfo.DNS.Nameservers, ","),
+		DNSSuffix:      strings.Join(epInfo.DNS.Search, ","),
+		GatewayAddress: GetIpString(&epInfo.Gateway),
+		IPAddress:      epInfo.IpAddress,
+		Policies:       n.GetHNSEndpointPolicies(),
+	}
+	return hnsEndpoint, nil
+}
+
+// RemoveHnsEndpoint detaches the given name endpoint from container specified by containerID,
+// or removes the given name endpoint completely.
+func RemoveHnsEndpoint(epName string, netns string, containerID string) error {
 	if len(netns) == 0 {
 		return nil
 	}
 
 	hnsEndpoint, err := hcsshim.GetHNSEndpointByName(epName)
-
-	if hcsshim.IsNotExist(err) {
-		return nil
-	} else if err != nil {
+	if err != nil {
+		if hcsshim.IsNotExist(err) {
+			return nil
+		}
 		return errors.Annotatef(err, "failed to find HNSEndpoint %s", epName)
 	}
 
+	// for shared endpoint, detach it from the container
 	if netns != pauseContainerNetNS {
-		// Shared endpoint removal. Do not remove the endpoint.
-		hnsEndpoint.ContainerDetach(containerID)
+		_ = hnsEndpoint.ContainerDetach(containerID)
 		return nil
 	}
 
-	// Do not consider this as failure, else this would leak endpoints
-	hcsshim.HotDetachEndpoint(containerID, hnsEndpoint.Id)
-
-	// Do not return error
-	hnsEndpoint.Delete()
-
+	// for removing the endpoint completely, hot detach is used at first
+	_ = hcsshim.HotDetachEndpoint(containerID, hnsEndpoint.Id)
+	_, _ = hnsEndpoint.Delete()
 	return nil
 }
 
-type EndpointMakerFunc func() (*hcsshim.HNSEndpoint, error)
+type HnsEndpointMakerFunc func() (*hcsshim.HNSEndpoint, error)
 
-// ProvisionEndpoint provisions an endpoint to a container specified by containerID.
-// If an endpoint already exists, the endpoint is reused.
-// This call is idempotent
-func ProvisionEndpoint(epName string, expectedNetworkId string, containerID string, netns string, makeEndpoint EndpointMakerFunc) (*hcsshim.HNSEndpoint, error) {
-	// On the second add call we expect that the endpoint already exists. If it
-	// does not then we should return an error.
-	if netns != pauseContainerNetNS {
-		_, err := hcsshim.GetHNSEndpointByName(epName)
-		if err != nil {
+// AddHnsEndpoint attaches an HNSEndpoint to a container specified by containerID.
+func AddHnsEndpoint(epName string, expectedNetworkId string, containerID string, netns string, makeEndpoint HnsEndpointMakerFunc) (*hcsshim.HNSEndpoint, error) {
+	hnsEndpoint, err := hcsshim.GetHNSEndpointByName(epName)
+	if err != nil {
+		if !hcsshim.IsNotExist(err) {
 			return nil, errors.Annotatef(err, "failed to find HNSEndpoint %s", epName)
 		}
 	}
 
-	// check if endpoint already exists
-	createEndpoint := true
-	hnsEndpoint, err := hcsshim.GetHNSEndpointByName(epName)
-	if hnsEndpoint != nil && strings.EqualFold(hnsEndpoint.VirtualNetwork, expectedNetworkId) {
-		createEndpoint = false
+	// for shared endpoint, we expect that the endpoint already exists
+	if netns != pauseContainerNetNS {
+		if hnsEndpoint == nil {
+			return nil, errors.Annotatef(err, "failed to find HNSEndpoint %s", epName)
+		}
 	}
 
-	if createEndpoint {
-		if hnsEndpoint != nil {
-			if _, err = hnsEndpoint.Delete(); err != nil {
-				return nil, errors.Annotate(err, "failed to delete the stale HNSEndpoint")
+	// verify the existing endpoint is corrupted or not
+	if hnsEndpoint != nil {
+		if !strings.EqualFold(hnsEndpoint.VirtualNetwork, expectedNetworkId) {
+			if _, err := hnsEndpoint.Delete(); err != nil {
+				return nil, errors.Annotatef(err, "failed to delete corrupted HNSEndpoint %s", epName)
 			}
+			hnsEndpoint = nil
 		}
+	}
 
+	// create endpoint if not found
+	var isNewEndpoint bool
+	if hnsEndpoint == nil {
 		if hnsEndpoint, err = makeEndpoint(); err != nil {
 			return nil, errors.Annotate(err, "failed to make a new HNSEndpoint")
 		}
-
 		if hnsEndpoint, err = hnsEndpoint.Create(); err != nil {
 			return nil, errors.Annotate(err, "failed to create the new HNSEndpoint")
 		}
-
+		isNewEndpoint = true
 	}
 
-	// hot attach
+	// attach to container
 	if err := hcsshim.HotAttachEndpoint(containerID, hnsEndpoint.Id); err != nil {
-		if createEndpoint {
-			err := DeprovisionEndpoint(epName, netns, containerID)
-			if err != nil {
-				return nil, errors.Annotatef(err, "failed to Deprovsion after HotAttach failure")
+		if isNewEndpoint {
+			if err := RemoveHnsEndpoint(epName, netns, containerID); err != nil {
+				return nil, errors.Annotatef(err, "failed to remove the new HNSEndpoint %s after attaching container %s failure", hnsEndpoint.Id, containerID)
 			}
-		}
-		if hcsshim.ErrComputeSystemDoesNotExist == err {
+		} else if hcsshim.ErrComputeSystemDoesNotExist == err {
 			return hnsEndpoint, nil
 		}
-		return nil, err
+		return nil, errors.Annotatef(err, "failed to attach container %s to HNSEndpoint %s", containerID, hnsEndpoint.Id)
 	}
-
 	return hnsEndpoint, nil
 }
 
-type HcnEndpointMakerFunc func() (*hcn.HostComputeEndpoint, error)
-
-func AddHcnEndpoint(epName string, expectedNetworkId string, namespace string,
-	makeEndpoint HcnEndpointMakerFunc) (*hcn.HostComputeEndpoint, error) {
-
-	hcnEndpoint, err := makeEndpoint()
-	if err != nil {
-		return nil, errors.Annotate(err, "failed to make a new HNSEndpoint")
-	}
-
-	if hcnEndpoint, err = hcnEndpoint.Create(); err != nil {
-		return nil, errors.Annotate(err, "failed to create the new HNSEndpoint")
-	}
-
-	err = hcn.AddNamespaceEndpoint(namespace, hcnEndpoint.Id)
-	if err != nil {
-		err := RemoveHcnEndpoint(epName)
-		if err != nil {
-			return nil, errors.Annotatef(err, "failed to Remove Endpoint after AddNamespaceEndpoint failure")
-		}
-		return nil, errors.Annotate(err, "failed to Add endpoint to namespace")
-	}
-	return hcnEndpoint, nil
-
-}
-
-// ConstructResult constructs the CNI result for the endpoint
-func ConstructResult(hnsNetwork *hcsshim.HNSNetwork, hnsEndpoint *hcsshim.HNSEndpoint) (*current.Result, error) {
+// ConstructHnsResult constructs the CNI result for the HNSEndpoint.
+func ConstructHnsResult(hnsNetwork *hcsshim.HNSNetwork, hnsEndpoint *hcsshim.HNSEndpoint) (*current.Result, error) {
 	resultInterface := &current.Interface{
 		Name: hnsEndpoint.Name,
 		Mac:  hnsEndpoint.MacAddress,
@@ -286,51 +201,140 @@ func ConstructResult(hnsNetwork *hcsshim.HNSNetwork, hnsEndpoint *hcsshim.HNSEnd
 		return nil, errors.Annotatef(err, "failed to parse CIDR from %s", hnsNetwork.Subnets[0].AddressPrefix)
 	}
 
-	var ipVersion string
-	if ipv4 := hnsEndpoint.IPAddress.To4(); ipv4 != nil {
-		ipVersion = "4"
-	} else if ipv6 := hnsEndpoint.IPAddress.To16(); ipv6 != nil {
-		ipVersion = "6"
-	} else {
-		return nil, fmt.Errorf("IPAddress of HNSEndpoint %s isn't a valid ipv4 or ipv6 Address", hnsEndpoint.Name)
-	}
-
 	resultIPConfig := &current.IPConfig{
-		Version: ipVersion,
 		Address: net.IPNet{
 			IP:   hnsEndpoint.IPAddress,
 			Mask: ipSubnet.Mask},
 		Gateway: net.ParseIP(hnsEndpoint.GatewayAddress),
 	}
-	result := &current.Result{}
-	result.Interfaces = []*current.Interface{resultInterface}
-	result.IPs = []*current.IPConfig{resultIPConfig}
-	result.DNS = types.DNS{
-		Search:      strings.Split(hnsEndpoint.DNSSuffix, ","),
-		Nameservers: strings.Split(hnsEndpoint.DNSServerList, ","),
+	result := &current.Result{
+		CNIVersion: current.ImplementedSpecVersion,
+		Interfaces: []*current.Interface{resultInterface},
+		IPs:        []*current.IPConfig{resultIPConfig},
+		DNS: types.DNS{
+			Search:      strings.Split(hnsEndpoint.DNSSuffix, ","),
+			Nameservers: strings.Split(hnsEndpoint.DNSServerList, ","),
+		},
 	}
 
 	return result, nil
 }
 
-// This version follows the v2 workflow of removing the endpoint from the namespace and deleting it
+// GenerateHcnEndpoint generates a HostComputeEndpoint with given info and config.
+func GenerateHcnEndpoint(epInfo *EndpointInfo, n *NetConf) (*hcn.HostComputeEndpoint, error) {
+	// run the IPAM plugin and get back the config to apply
+	hcnEndpoint, err := hcn.GetEndpointByName(epInfo.EndpointName)
+	if err != nil && !hcn.IsNotFoundError(err) {
+		return nil, errors.Annotatef(err, "failed to get HostComputeEndpoint %s", epInfo.EndpointName)
+	}
+
+	// verify the existing endpoint is corrupted or not
+	if hcnEndpoint != nil {
+		if strings.EqualFold(hcnEndpoint.HostComputeNetwork, epInfo.NetworkId) {
+			return nil, fmt.Errorf("HostComputeNetwork %s is already existed", epInfo.EndpointName)
+		}
+		// remove endpoint if corrupted
+		if err := hcnEndpoint.Delete(); err != nil {
+			return nil, errors.Annotatef(err, "failed to delete corrupted HostComputeEndpoint %s", epInfo.EndpointName)
+		}
+	}
+
+	if n.LoopbackDSR {
+		n.ApplyLoopbackDSRPolicy(&epInfo.IpAddress)
+	}
+	hcnEndpoint = &hcn.HostComputeEndpoint{
+		SchemaVersion: hcn.SchemaVersion{
+			Major: 2,
+			Minor: 0,
+		},
+		Name:               epInfo.EndpointName,
+		HostComputeNetwork: epInfo.NetworkId,
+		Dns: hcn.Dns{
+			Domain:     epInfo.DNS.Domain,
+			Search:     epInfo.DNS.Search,
+			ServerList: epInfo.DNS.Nameservers,
+			Options:    epInfo.DNS.Options,
+		},
+		Routes: []hcn.Route{
+			{
+				NextHop:           GetIpString(&epInfo.Gateway),
+				DestinationPrefix: GetDefaultDestinationPrefix(&epInfo.Gateway),
+			},
+		},
+		IpConfigurations: []hcn.IpConfig{
+			{
+				IpAddress: GetIpString(&epInfo.IpAddress),
+			},
+		},
+		Policies: n.GetHostComputeEndpointPolicies(),
+	}
+	return hcnEndpoint, nil
+}
+
+// RemoveHcnEndpoint removes the given name endpoint from namespace.
 func RemoveHcnEndpoint(epName string) error {
 	hcnEndpoint, err := hcn.GetEndpointByName(epName)
-	if hcn.IsNotFoundError(err) {
-		return nil
-	} else if err != nil {
-		_ = fmt.Errorf("[win-cni] Failed to find endpoint %v, err:%v", epName, err)
-		return err
-	}
-	if hcnEndpoint != nil {
-		err = hcnEndpoint.Delete()
-		if err != nil {
-			return fmt.Errorf("[win-cni] Failed to delete endpoint %v, err:%v", epName, err)
+	if err != nil {
+		if hcn.IsNotFoundError(err) {
+			return nil
 		}
+		return errors.Annotatef(err, "failed to find HostComputeEndpoint %s", epName)
+	}
+
+	err = hcnEndpoint.Delete()
+	if err != nil {
+		return errors.Annotatef(err, "failed to remove HostComputeEndpoint %s", epName)
 	}
 	return nil
 }
 
+type HcnEndpointMakerFunc func() (*hcn.HostComputeEndpoint, error)
+
+// AddHcnEndpoint attaches a HostComputeEndpoint to the given namespace.
+func AddHcnEndpoint(epName string, expectedNetworkId string, namespace string, makeEndpoint HcnEndpointMakerFunc) (*hcn.HostComputeEndpoint, error) {
+	hcnEndpoint, err := hcn.GetEndpointByName(epName)
+	if err != nil {
+		if !hcn.IsNotFoundError(err) {
+			return nil, errors.Annotatef(err, "failed to find HostComputeEndpoint %s", epName)
+		}
+	}
+
+	// verify the existing endpoint is corrupted or not
+	if hcnEndpoint != nil {
+		if !strings.EqualFold(hcnEndpoint.HostComputeNetwork, expectedNetworkId) {
+			if err := hcnEndpoint.Delete(); err != nil {
+				return nil, errors.Annotatef(err, "failed to delete corrupted HostComputeEndpoint %s", epName)
+			}
+			hcnEndpoint = nil
+		}
+	}
+
+	// create endpoint if not found
+	var isNewEndpoint bool
+	if hcnEndpoint == nil {
+		if hcnEndpoint, err = makeEndpoint(); err != nil {
+			return nil, errors.Annotate(err, "failed to make a new HostComputeEndpoint")
+		}
+		if hcnEndpoint, err = hcnEndpoint.Create(); err != nil {
+			return nil, errors.Annotate(err, "failed to create the new HostComputeEndpoint")
+		}
+		isNewEndpoint = true
+	}
+
+	// add to namespace
+	err = hcn.AddNamespaceEndpoint(namespace, hcnEndpoint.Id)
+	if err != nil {
+		if isNewEndpoint {
+			if err := RemoveHcnEndpoint(epName); err != nil {
+				return nil, errors.Annotatef(err, "failed to remove the new HostComputeEndpoint %s after adding HostComputeNamespace %s failure", epName, namespace)
+			}
+		}
+		return nil, errors.Annotatef(err, "failed to add HostComputeEndpoint %s to HostComputeNamespace %s", epName, namespace)
+	}
+	return hcnEndpoint, nil
+}
+
+// ConstructHcnResult constructs the CNI result for the HostComputeEndpoint.
 func ConstructHcnResult(hcnNetwork *hcn.HostComputeNetwork, hcnEndpoint *hcn.HostComputeEndpoint) (*current.Result, error) {
 	resultInterface := &current.Interface{
 		Name: hcnEndpoint.Name,
@@ -341,29 +345,23 @@ func ConstructHcnResult(hcnNetwork *hcn.HostComputeNetwork, hcnEndpoint *hcn.Hos
 		return nil, err
 	}
 
-	var ipVersion string
 	ipAddress := net.ParseIP(hcnEndpoint.IpConfigurations[0].IpAddress)
-	if ipv4 := ipAddress.To4(); ipv4 != nil {
-		ipVersion = "4"
-	} else if ipv6 := ipAddress.To16(); ipv6 != nil {
-		ipVersion = "6"
-	} else {
-		return nil, fmt.Errorf("[win-cni] The IPAddress of hnsEndpoint isn't a valid ipv4 or ipv6 Address.")
-	}
-
 	resultIPConfig := &current.IPConfig{
-		Version: ipVersion,
 		Address: net.IPNet{
 			IP:   ipAddress,
 			Mask: ipSubnet.Mask},
 		Gateway: net.ParseIP(hcnEndpoint.Routes[0].NextHop),
 	}
-	result := &current.Result{}
-	result.Interfaces = []*current.Interface{resultInterface}
-	result.IPs = []*current.IPConfig{resultIPConfig}
-	result.DNS = types.DNS{
-		Search:      hcnEndpoint.Dns.Search,
-		Nameservers: hcnEndpoint.Dns.ServerList,
+	result := &current.Result{
+		CNIVersion: current.ImplementedSpecVersion,
+		Interfaces: []*current.Interface{resultInterface},
+		IPs:        []*current.IPConfig{resultIPConfig},
+		DNS: types.DNS{
+			Search:      hcnEndpoint.Dns.Search,
+			Nameservers: hcnEndpoint.Dns.ServerList,
+			Options:     hcnEndpoint.Dns.Options,
+			Domain:      hcnEndpoint.Dns.Domain,
+		},
 	}
 
 	return result, nil

pkg/hns/hns_suite_windows_test.go

@@ -1,13 +0,0 @@
-package hns_test
-
-import (
-	"testing"
-
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-)
-
-func TestHns(t *testing.T) {
-	RegisterFailHandler(Fail)
-	RunSpecs(t, "Hns Suite")
-}

pkg/hns/netconf_suite_windows_test.go

@@ -20,7 +20,7 @@ import (
 	"testing"
 )
 
-func TestHns(t *testing.T) {
+func TestNetConf(t *testing.T) {
 	RegisterFailHandler(Fail)
-	RunSpecs(t, "HNS NetConf Suite")
+	RunSpecs(t, "NetConf Suite")
 }

pkg/hns/netconf_windows.go

@@ -17,9 +17,10 @@ package hns
 import (
 	"bytes"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"net"
-
+	"strconv"
 	"strings"
 
 	"github.com/Microsoft/hcsshim/hcn"
@@ -30,16 +31,16 @@ import (
 // NetConf is the CNI spec
 type NetConf struct {
 	types.NetConf
-	// ApiVersion is either 1 or 2, which specifies which hns APIs to call
-	ApiVersion int `json:"ApiVersion"`
-	// V2 Api Policies
-	HcnPolicyArgs []hcn.EndpointPolicy `json:"HcnPolicyArgs,omitempty"`
-	// V1 Api Policies
-	Policies []policy `json:"policies,omitempty"`
-	// Options to be passed in by the runtime
+	// ApiVersion specifies the policies type of HNS or HCN, select one of [1, 2].
+	// HNS is the v1 API, which is the default version and applies to dockershim.
+	// HCN is the v2 API, which can leverage HostComputeNamespace and use in containerd.
+	ApiVersion int `json:"apiVersion,omitempty"`
+	// Policies specifies the policy list for HNSEndpoint or HostComputeEndpoint.
+	Policies []Policy `json:"policies,omitempty"`
+	// RuntimeConfig represents the options to be passed in by the runtime.
 	RuntimeConfig RuntimeConfig `json:"runtimeConfig"`
-	// If true, adds a policy to endpoints to support loopback direct server return
-	LoopbackDSR bool `json:"loopbackDSR"`
+	// LoopbackDSR specifies whether to support loopback direct server return.
+	LoopbackDSR bool `json:"loopbackDSR,omitempty"`
 }
 
 type RuntimeDNS struct {
@@ -54,42 +55,67 @@ type PortMapEntry struct {
 	HostIP        string `json:"hostIP,omitempty"`
 }
 
+// constants of the supported Windows Socket protocol,
+// ref to https://docs.microsoft.com/en-us/dotnet/api/system.net.sockets.protocoltype.
+var protocolEnums = map[string]uint32{
+	"icmpv4": 1,
+	"igmp":   2,
+	"tcp":    6,
+	"udp":    17,
+	"icmpv6": 58,
+}
+
+func (p *PortMapEntry) GetProtocolEnum() (uint32, error) {
+	var u, err = strconv.ParseUint(p.Protocol, 0, 10)
+	if err != nil {
+		var pe, exist = protocolEnums[strings.ToLower(p.Protocol)]
+		if !exist {
+			return 0, errors.New("invalid protocol supplied to port mapping policy")
+		}
+		return pe, nil
+	}
+	return uint32(u), nil
+}
+
 type RuntimeConfig struct {
 	DNS      RuntimeDNS     `json:"dns"`
 	PortMaps []PortMapEntry `json:"portMappings,omitempty"`
 }
 
-type policy struct {
+type Policy struct {
 	Name  string          `json:"name"`
 	Value json.RawMessage `json:"value"`
 }
 
-func GetDefaultDestinationPrefix(ip *net.IP) string {
-	destinationPrefix := "0.0.0.0/0"
-	if ipv6 := ip.To4(); ipv6 == nil {
-		destinationPrefix = "::/0"
+// GetHNSEndpointPolicies converts the configuration policies to HNSEndpoint policies.
+func (n *NetConf) GetHNSEndpointPolicies() []json.RawMessage {
+	result := make([]json.RawMessage, 0, len(n.Policies))
+	for _, p := range n.Policies {
+		if !strings.EqualFold(p.Name, "EndpointPolicy") {
+			continue
+		}
+		result = append(result, p.Value)
 	}
-	return destinationPrefix
+	return result
 }
 
-func (n *NetConf) ApplyLoopbackDSR(ip *net.IP) {
-	value := fmt.Sprintf(`"Destinations" : ["%s"]`, ip.String())
-	if n.ApiVersion == 2 {
-		hcnLoopbackRoute := hcn.EndpointPolicy{
-			Type:     "OutBoundNAT",
-			Settings: []byte(fmt.Sprintf("{%s}", value)),
+// GetHostComputeEndpointPolicies converts the configuration policies to HostComputeEndpoint policies.
+func (n *NetConf) GetHostComputeEndpointPolicies() []hcn.EndpointPolicy {
+	result := make([]hcn.EndpointPolicy, 0, len(n.Policies))
+	for _, p := range n.Policies {
+		if !strings.EqualFold(p.Name, "EndpointPolicy") {
+			continue
 		}
-		n.HcnPolicyArgs = append(n.HcnPolicyArgs, hcnLoopbackRoute)
-	} else {
-		hnsLoopbackRoute := policy{
-			Name:  "EndpointPolicy",
-			Value: []byte(fmt.Sprintf(`{"Type": "OutBoundNAT", %s}`, value)),
+		var policy hcn.EndpointPolicy
+		if err := json.Unmarshal(p.Value, &policy); err != nil {
+			continue
 		}
-		n.Policies = append(n.Policies, hnsLoopbackRoute)
+		result = append(result, policy)
 	}
+	return result
 }
 
-// If runtime dns values are there use that else use cni conf supplied dns
+// GetDNS returns the DNS values if they are there use that else use netconf supplied DNS.
 func (n *NetConf) GetDNS() types.DNS {
 	dnsResult := n.DNS
 	if len(n.RuntimeConfig.DNS.Nameservers) > 0 {
@@ -101,136 +127,222 @@ func (n *NetConf) GetDNS() types.DNS {
 	return dnsResult
 }
 
-// MarshalPolicies converts the Endpoint policies in Policies
-// to HNS specific policies as Json raw bytes
-func (n *NetConf) MarshalPolicies() []json.RawMessage {
-	if n.Policies == nil {
-		n.Policies = make([]policy, 0)
+// ApplyLoopbackDSRPolicy configures the given IP to support loopback DSR.
+func (n *NetConf) ApplyLoopbackDSRPolicy(ip *net.IP) {
+	if err := hcn.DSRSupported(); err != nil || ip == nil {
+		return
 	}
 
-	result := make([]json.RawMessage, 0, len(n.Policies))
-	for _, p := range n.Policies {
+	toPolicyValue := func(addr string) json.RawMessage {
+		if n.ApiVersion == 2 {
+			return bprintf(`{"Type": "OutBoundNAT", "Settings": {"Destinations": ["%s"]}}`, addr)
+		}
+		return bprintf(`{"Type": "OutBoundNAT", "Destinations": ["%s"]}`, addr)
+	}
+	ipBytes := []byte(ip.String())
+
+	// find OutBoundNAT policy
+	for i := range n.Policies {
+		p := &n.Policies[i]
 		if !strings.EqualFold(p.Name, "EndpointPolicy") {
 			continue
 		}
 
-		result = append(result, p.Value)
-	}
-
-	return result
-}
-
-// ApplyOutboundNatPolicy applies NAT Policy in VFP using HNS
-// Simultaneously an exception is added for the network that has to be Nat'd
-func (n *NetConf) ApplyOutboundNatPolicy(nwToNat string) {
-	if n.Policies == nil {
-		n.Policies = make([]policy, 0)
-	}
-
-	nwToNatBytes := []byte(nwToNat)
-
-	for i, p := range n.Policies {
-		if !strings.EqualFold(p.Name, "EndpointPolicy") {
+		// filter OutBoundNAT policy
+		typeValue, _ := jsonparser.GetUnsafeString(p.Value, "Type")
+		if typeValue != "OutBoundNAT" {
 			continue
 		}
 
-		typeValue, err := jsonparser.GetUnsafeString(p.Value, "Type")
-		if err != nil || len(typeValue) == 0 {
+		// parse destination address list
+		var (
+			destinationsValue []byte
+			dt                jsonparser.ValueType
+		)
+		if n.ApiVersion == 2 {
+			destinationsValue, dt, _, _ = jsonparser.Get(p.Value, "Settings", "Destinations")
+		} else {
+			destinationsValue, dt, _, _ = jsonparser.Get(p.Value, "Destinations")
+		}
+
+		// skip if Destinations/DestinationList field is not found
+		if dt == jsonparser.NotExist {
 			continue
 		}
 
-		if !strings.EqualFold(typeValue, "OutBoundNAT") {
-			continue
-		}
-
-		exceptionListValue, dt, _, _ := jsonparser.Get(p.Value, "ExceptionList")
-		// OutBoundNAT must with ExceptionList, so don't need to judge jsonparser.NotExist
+		// return if found the given address
 		if dt == jsonparser.Array {
-			buf := bytes.Buffer{}
-			buf.WriteString(`{"Type": "OutBoundNAT", "ExceptionList": [`)
-
-			jsonparser.ArrayEach(exceptionListValue, func(value []byte, dataType jsonparser.ValueType, offset int, err error) {
+			var found bool
+			_, _ = jsonparser.ArrayEach(destinationsValue, func(value []byte, dataType jsonparser.ValueType, offset int, err error) {
 				if dataType == jsonparser.String && len(value) != 0 {
-					if bytes.Compare(value, nwToNatBytes) != 0 {
-						buf.WriteByte('"')
-						buf.Write(value)
-						buf.WriteByte('"')
-						buf.WriteByte(',')
+					if bytes.Compare(value, ipBytes) == 0 {
+						found = true
 					}
 				}
 			})
-
-			buf.WriteString(`"` + nwToNat + `"]}`)
-
-			n.Policies[i] = policy{
-				Name:  "EndpointPolicy",
-				Value: buf.Bytes(),
-			}
-		} else {
-			n.Policies[i] = policy{
-				Name:  "EndpointPolicy",
-				Value: []byte(`{"Type": "OutBoundNAT", "ExceptionList": ["` + nwToNat + `"]}`),
+			if found {
+				return
 			}
 		}
-
-		return
 	}
 
-	// didn't find the policyArg, add it
-	n.Policies = append(n.Policies, policy{
+	// or add a new OutBoundNAT if not found
+	n.Policies = append(n.Policies, Policy{
 		Name:  "EndpointPolicy",
-		Value: []byte(`{"Type": "OutBoundNAT", "ExceptionList": ["` + nwToNat + `"]}`),
+		Value: toPolicyValue(ip.String()),
 	})
 }
 
-// ApplyDefaultPAPolicy is used to configure a endpoint PA policy in HNS
-func (n *NetConf) ApplyDefaultPAPolicy(paAddress string) {
-	if n.Policies == nil {
-		n.Policies = make([]policy, 0)
+// ApplyOutboundNatPolicy applies the sNAT policy in HNS/HCN and configures the given CIDR as an exception.
+func (n *NetConf) ApplyOutboundNatPolicy(exceptionCIDR string) {
+	if exceptionCIDR == "" {
+		return
 	}
 
-	// if its already present, leave untouched
-	for i, p := range n.Policies {
+	toPolicyValue := func(cidr ...string) json.RawMessage {
+		if n.ApiVersion == 2 {
+			return bprintf(`{"Type": "OutBoundNAT", "Settings": {"Exceptions": ["%s"]}}`, strings.Join(cidr, `","`))
+		}
+		return bprintf(`{"Type": "OutBoundNAT", "ExceptionList": ["%s"]}`, strings.Join(cidr, `","`))
+	}
+	exceptionCIDRBytes := []byte(exceptionCIDR)
+
+	// find OutBoundNAT policy
+	for i := range n.Policies {
+		p := &n.Policies[i]
 		if !strings.EqualFold(p.Name, "EndpointPolicy") {
 			continue
 		}
 
-		paValue, dt, _, _ := jsonparser.Get(p.Value, "PA")
+		// filter OutBoundNAT policy
+		typeValue, _ := jsonparser.GetUnsafeString(p.Value, "Type")
+		if typeValue != "OutBoundNAT" {
+			continue
+		}
+
+		// parse exception CIDR list
+		var (
+			exceptionsValue []byte
+			dt              jsonparser.ValueType
+		)
+		if n.ApiVersion == 2 {
+			exceptionsValue, dt, _, _ = jsonparser.Get(p.Value, "Settings", "Exceptions")
+		} else {
+			exceptionsValue, dt, _, _ = jsonparser.Get(p.Value, "ExceptionList")
+		}
+
+		// skip if Exceptions/ExceptionList field is not found
 		if dt == jsonparser.NotExist {
 			continue
-		} else if dt == jsonparser.String && len(paValue) != 0 {
-			// found it, don't override
-			return
 		}
 
-		n.Policies[i] = policy{
-			Name:  "EndpointPolicy",
-			Value: []byte(`{"Type": "PA", "PA": "` + paAddress + `"}`),
+		// return if found the given CIDR
+		if dt == jsonparser.Array {
+			var found bool
+			_, _ = jsonparser.ArrayEach(exceptionsValue, func(value []byte, dataType jsonparser.ValueType, offset int, err error) {
+				if dataType == jsonparser.String && len(value) != 0 {
+					if bytes.Compare(value, exceptionCIDRBytes) == 0 {
+						found = true
+					}
+				}
+			})
+			if found {
+				return
+			}
 		}
-		return
 	}
 
-	// didn't find the policyArg, add it
-	n.Policies = append(n.Policies, policy{
+	// or add a new OutBoundNAT if not found
+	n.Policies = append(n.Policies, Policy{
 		Name:  "EndpointPolicy",
-		Value: []byte(`{"Type": "PA", "PA": "` + paAddress + `"}`),
+		Value: toPolicyValue(exceptionCIDR),
 	})
 }
 
-// ApplyPortMappingPolicy is used to configure HostPort<>ContainerPort mapping in HNS
-func (n *NetConf) ApplyPortMappingPolicy(portMappings []PortMapEntry) {
-	if portMappings == nil {
+// ApplyDefaultPAPolicy applies an endpoint PA policy in HNS/HCN.
+func (n *NetConf) ApplyDefaultPAPolicy(address string) {
+	if address == "" {
 		return
 	}
 
-	if n.Policies == nil {
-		n.Policies = make([]policy, 0)
+	toPolicyValue := func(addr string) json.RawMessage {
+		if n.ApiVersion == 2 {
+			return bprintf(`{"Type": "ProviderAddress", "Settings": {"ProviderAddress": "%s"}}`, addr)
+		}
+		return bprintf(`{"Type": "PA", "PA": "%s"}`, addr)
+	}
+	addressBytes := []byte(address)
+
+	// find ProviderAddress policy
+	for i := range n.Policies {
+		p := &n.Policies[i]
+		if !strings.EqualFold(p.Name, "EndpointPolicy") {
+			continue
+		}
+
+		// filter ProviderAddress policy
+		typeValue, _ := jsonparser.GetUnsafeString(p.Value, "Type")
+		if typeValue != "PA" && typeValue != "ProviderAddress" {
+			continue
+		}
+
+		// parse provider address
+		var (
+			paValue []byte
+			dt      jsonparser.ValueType
+		)
+		if n.ApiVersion == 2 {
+			paValue, dt, _, _ = jsonparser.Get(p.Value, "Settings", "ProviderAddress")
+		} else {
+			paValue, dt, _, _ = jsonparser.Get(p.Value, "PA")
+		}
+
+		// skip if ProviderAddress/PA field is not found
+		if dt == jsonparser.NotExist {
+			continue
+		}
+
+		// return if found the given address
+		if dt == jsonparser.String && bytes.Compare(paValue, addressBytes) == 0 {
+			return
+		}
 	}
 
-	for _, portMapping := range portMappings {
-		n.Policies = append(n.Policies, policy{
+	// or add a new ProviderAddress if not found
+	n.Policies = append(n.Policies, Policy{
+		Name:  "EndpointPolicy",
+		Value: toPolicyValue(address),
+	})
+}
+
+// ApplyPortMappingPolicy applies the host/container port mapping policies in HNS/HCN.
+func (n *NetConf) ApplyPortMappingPolicy(portMappings []PortMapEntry) {
+	if len(portMappings) == 0 {
+		return
+	}
+
+	toPolicyValue := func(p *PortMapEntry) json.RawMessage {
+		if n.ApiVersion == 2 {
+			var protocolEnum, _ = p.GetProtocolEnum()
+			return bprintf(`{"Type": "PortMapping", "Settings": {"InternalPort": %d, "ExternalPort": %d, "Protocol": %d, "VIP": "%s"}}`, p.ContainerPort, p.HostPort, protocolEnum, p.HostIP)
+		}
+		return bprintf(`{"Type": "NAT", "InternalPort": %d, "ExternalPort": %d, "Protocol": "%s"}`, p.ContainerPort, p.HostPort, p.Protocol)
+	}
+
+	for i := range portMappings {
+		p := &portMappings[i]
+		// skip the invalid protocol mapping
+		if _, err := p.GetProtocolEnum(); err != nil {
+			continue
+		}
+		n.Policies = append(n.Policies, Policy{
 			Name:  "EndpointPolicy",
-			Value: []byte(fmt.Sprintf(`{"Type": "NAT", "InternalPort": %d, "ExternalPort": %d, "Protocol": "%s"}`, portMapping.ContainerPort, portMapping.HostPort, portMapping.Protocol)),
+			Value: toPolicyValue(p),
 		})
 	}
 }
+
+// bprintf is similar to fmt.Sprintf and returns a byte array as result.
+func bprintf(format string, a ...interface{}) []byte {
+	return []byte(fmt.Sprintf(format, a...))
+}

pkg/hns/netconf_windows_test.go

@@ -15,221 +15,585 @@ package hns
 
 import (
 	"encoding/json"
+	"net"
 
+	"github.com/Microsoft/hcsshim/hcn"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 )
 
-var _ = Describe("HNS NetConf", func() {
-	Describe("ApplyOutBoundNATPolicy", func() {
-		Context("when not set by user", func() {
-			It("sets it by adding a policy", func() {
+var _ = Describe("NetConf", func() {
+	Describe("ApplyLoopbackDSRPolicy", func() {
+		Context("via v1 api", func() {
+			var n NetConf
+			BeforeEach(func() {
+				n = NetConf{}
+			})
 
-				// apply it
-				n := NetConf{}
-				n.ApplyOutboundNatPolicy("192.168.0.0/16")
+			It("filter out duplicated IP", func() {
+				// mock duplicated IP
+				ip := net.ParseIP("172.16.0.12")
+				n.ApplyLoopbackDSRPolicy(&ip)
+				n.ApplyLoopbackDSRPolicy(&ip)
 
+				// only one policy
 				addlArgs := n.Policies
 				Expect(addlArgs).Should(HaveLen(1))
 
+				// normal type judgement
 				policy := addlArgs[0]
 				Expect(policy.Name).Should(Equal("EndpointPolicy"))
-
 				value := make(map[string]interface{})
 				json.Unmarshal(policy.Value, &value)
-
 				Expect(value).Should(HaveKey("Type"))
-				Expect(value).Should(HaveKey("ExceptionList"))
 				Expect(value["Type"]).Should(Equal("OutBoundNAT"))
+				Expect(value).Should(HaveKey("Destinations"))
 
-				exceptionList := value["ExceptionList"].([]interface{})
-				Expect(exceptionList).Should(HaveLen(1))
-				Expect(exceptionList[0].(string)).Should(Equal("192.168.0.0/16"))
+				// and only one item
+				destinationList := value["Destinations"].([]interface{})
+				Expect(destinationList).Should(HaveLen(1))
+				Expect(destinationList[0].(string)).Should(Equal("172.16.0.12"))
+			})
+
+			It("append different IP", func() {
+				// mock different IP
+				ip1 := net.ParseIP("172.16.0.12")
+				n.ApplyLoopbackDSRPolicy(&ip1)
+				ip2 := net.ParseIP("172.16.0.13")
+				n.ApplyLoopbackDSRPolicy(&ip2)
+
+				// will be two policies
+				addlArgs := n.Policies
+				Expect(addlArgs).Should(HaveLen(2))
+
+				// normal type judgement
+				policy := addlArgs[1] // pick second item
+				Expect(policy.Name).Should(Equal("EndpointPolicy"))
+				value := make(map[string]interface{})
+				json.Unmarshal(policy.Value, &value)
+				Expect(value).Should(HaveKey("Type"))
+				Expect(value["Type"]).Should(Equal("OutBoundNAT"))
+				Expect(value).Should(HaveKey("Destinations"))
+
+				// only one item
+				destinationList := value["Destinations"].([]interface{})
+				Expect(destinationList).Should(HaveLen(1))
+				Expect(destinationList[0].(string)).Should(Equal("172.16.0.13"))
 			})
 		})
 
-		Context("when set by user", func() {
-			It("appends exceptions to the existing policy", func() {
-				// first set it
-				n := NetConf{}
-				n.ApplyOutboundNatPolicy("192.168.0.0/16")
+		Context("via v2 api", func() {
+			var n NetConf
+			BeforeEach(func() {
+				n = NetConf{ApiVersion: 2}
+			})
 
-				// then attempt to update it
-				n.ApplyOutboundNatPolicy("10.244.0.0/16")
+			It("filter out duplicated IP", func() {
+				// mock duplicated IP
+				ip := net.ParseIP("172.16.0.12")
+				n.ApplyLoopbackDSRPolicy(&ip)
+				n.ApplyLoopbackDSRPolicy(&ip)
 
-				// it should be unchanged!
+				// only one policy
 				addlArgs := n.Policies
 				Expect(addlArgs).Should(HaveLen(1))
 
+				// normal type judgement
 				policy := addlArgs[0]
 				Expect(policy.Name).Should(Equal("EndpointPolicy"))
-
-				var value map[string]interface{}
+				value := make(map[string]interface{})
 				json.Unmarshal(policy.Value, &value)
-
 				Expect(value).Should(HaveKey("Type"))
-				Expect(value).Should(HaveKey("ExceptionList"))
 				Expect(value["Type"]).Should(Equal("OutBoundNAT"))
+				Expect(value).Should(HaveKey("Settings"))
 
+				// and only one item
+				settings := value["Settings"].(map[string]interface{})
+				destinationList := settings["Destinations"].([]interface{})
+				Expect(destinationList).Should(HaveLen(1))
+				Expect(destinationList[0].(string)).Should(Equal("172.16.0.12"))
+			})
+
+			It("append different IP", func() {
+				// mock different IP
+				ip1 := net.ParseIP("172.16.0.12")
+				n.ApplyLoopbackDSRPolicy(&ip1)
+				ip2 := net.ParseIP("172.16.0.13")
+				n.ApplyLoopbackDSRPolicy(&ip2)
+
+				// will be two policies
+				addlArgs := n.Policies
+				Expect(addlArgs).Should(HaveLen(2))
+
+				// normal type judgement
+				policy := addlArgs[1] // pick second item
+				Expect(policy.Name).Should(Equal("EndpointPolicy"))
+				value := make(map[string]interface{})
+				json.Unmarshal(policy.Value, &value)
+				Expect(value).Should(HaveKey("Type"))
+				Expect(value["Type"]).Should(Equal("OutBoundNAT"))
+				Expect(value).Should(HaveKey("Settings"))
+
+				// only one item
+				settings := value["Settings"].(map[string]interface{})
+				destinationList := settings["Destinations"].([]interface{})
+				Expect(destinationList).Should(HaveLen(1))
+				Expect(destinationList[0].(string)).Should(Equal("172.16.0.13"))
+			})
+		})
+	})
+
+	Describe("ApplyOutBoundNATPolicy", func() {
+		Context("via v1 api", func() {
+			var n NetConf
+			BeforeEach(func() {
+				n = NetConf{}
+			})
+
+			It("append different IP", func() {
+				// mock different IP
+				n.ApplyOutboundNatPolicy("192.168.0.0/16")
+				n.ApplyOutboundNatPolicy("10.244.0.0/16")
+
+				// will be two policies
+				addlArgs := n.Policies
+				Expect(addlArgs).Should(HaveLen(2))
+
+				// normal type judgement
+				policy := addlArgs[1] // pick second item
+				Expect(policy.Name).Should(Equal("EndpointPolicy"))
+				value := make(map[string]interface{})
+				json.Unmarshal(policy.Value, &value)
+				Expect(value).Should(HaveKey("Type"))
+				Expect(value["Type"]).Should(Equal("OutBoundNAT"))
+				Expect(value).Should(HaveKey("ExceptionList"))
+
+				// but get two items
 				exceptionList := value["ExceptionList"].([]interface{})
-				Expect(exceptionList).Should(HaveLen(2))
-				Expect(exceptionList[0].(string)).Should(Equal("192.168.0.0/16"))
-				Expect(exceptionList[1].(string)).Should(Equal("10.244.0.0/16"))
+				Expect(exceptionList).Should(HaveLen(1))
+				Expect(exceptionList[0].(string)).Should(Equal("10.244.0.0/16"))
+			})
+
+			It("append a new one if there is not an exception OutBoundNAT policy", func() {
+				// mock different OutBoundNAT routes
+				n.Policies = []Policy{
+					{
+						Name:  "EndpointPolicy",
+						Value: bprintf(`{"Type": "OutBoundNAT", "OtherList": []}`),
+					},
+				}
+				n.ApplyOutboundNatPolicy("10.244.0.0/16")
+
+				// has two policies
+				addlArgs := n.Policies
+				Expect(addlArgs).Should(HaveLen(2))
+
+				// normal type judgement
+				policy := addlArgs[0]
+				Expect(policy.Name).Should(Equal("EndpointPolicy"))
+				value := make(map[string]interface{})
+				json.Unmarshal(policy.Value, &value)
+				Expect(value).Should(HaveKey("Type"))
+				Expect(value["Type"]).Should(Equal("OutBoundNAT"))
+				Expect(value).Should(HaveKey("OtherList"))
+				policy = addlArgs[1]
+				value = make(map[string]interface{})
+				json.Unmarshal(policy.Value, &value)
+				Expect(value).Should(HaveKey("Type"))
+				Expect(value["Type"]).Should(Equal("OutBoundNAT"))
+				Expect(value).Should(HaveKey("ExceptionList"))
+
+				// only get one item
+				exceptionList := value["ExceptionList"].([]interface{})
+				Expect(exceptionList).Should(HaveLen(1))
+				Expect(exceptionList[0].(string)).Should(Equal("10.244.0.0/16"))
+			})
+
+			It("nothing to do if CIDR is blank", func() {
+				// mock different OutBoundNAT routes
+				n.Policies = []Policy{
+					{
+						Name:  "EndpointPolicy",
+						Value: bprintf(`{"Type": "OutBoundNAT", "ExceptionList": []}`),
+					},
+				}
+				n.ApplyOutboundNatPolicy("")
+
+				// only one policy
+				addlArgs := n.Policies
+				Expect(addlArgs).Should(HaveLen(1))
+
+				// normal type judgement
+				policy := addlArgs[0]
+				Expect(policy.Name).Should(Equal("EndpointPolicy"))
+				value := make(map[string]interface{})
+				json.Unmarshal(policy.Value, &value)
+				Expect(value).Should(HaveKey("Type"))
+				Expect(value["Type"]).Should(Equal("OutBoundNAT"))
+				Expect(value).Should(HaveKey("ExceptionList"))
+
+				// empty list
+				Expect(value["ExceptionList"]).ShouldNot(BeNil())
+				Expect(value["ExceptionList"]).Should(HaveLen(0))
+			})
+		})
+
+		Context("via v2 api", func() {
+			var n NetConf
+			BeforeEach(func() {
+				n = NetConf{ApiVersion: 2}
+			})
+
+			It("append different IP", func() {
+				// mock different IP
+				n.ApplyOutboundNatPolicy("192.168.0.0/16")
+				n.ApplyOutboundNatPolicy("10.244.0.0/16")
+
+				// will be two policies
+				addlArgs := n.Policies
+				Expect(addlArgs).Should(HaveLen(2))
+
+				// normal type judgement
+				policy := addlArgs[1] // pick second item
+				Expect(policy.Name).Should(Equal("EndpointPolicy"))
+				value := make(map[string]interface{})
+				json.Unmarshal(policy.Value, &value)
+				Expect(value).Should(HaveKey("Type"))
+				Expect(value["Type"]).Should(Equal("OutBoundNAT"))
+				Expect(value).Should(HaveKey("Settings"))
+
+				// but get two items
+				settings := value["Settings"].(map[string]interface{})
+				exceptionList := settings["Exceptions"].([]interface{})
+				Expect(exceptionList).Should(HaveLen(1))
+				Expect(exceptionList[0].(string)).Should(Equal("10.244.0.0/16"))
+			})
+
+			It("append a new one if there is not an exception OutBoundNAT policy", func() {
+				// mock different OutBoundNAT routes
+				n.Policies = []Policy{
+					{
+						Name:  "EndpointPolicy",
+						Value: bprintf(`{"Type": "OutBoundNAT", "Settings": {"Others": []}}`),
+					},
+				}
+				n.ApplyOutboundNatPolicy("10.244.0.0/16")
+
+				// has two policies
+				addlArgs := n.Policies
+				Expect(addlArgs).Should(HaveLen(2))
+
+				// normal type judgement
+				policy := addlArgs[0]
+				Expect(policy.Name).Should(Equal("EndpointPolicy"))
+				value := make(map[string]interface{})
+				json.Unmarshal(policy.Value, &value)
+				Expect(value).Should(HaveKey("Type"))
+				Expect(value["Type"]).Should(Equal("OutBoundNAT"))
+				Expect(value).Should(HaveKey("Settings"))
+				Expect(value["Settings"]).Should(HaveKey("Others"))
+				policy = addlArgs[1]
+				value = make(map[string]interface{})
+				json.Unmarshal(policy.Value, &value)
+				Expect(value).Should(HaveKey("Type"))
+				Expect(value["Type"]).Should(Equal("OutBoundNAT"))
+				Expect(value).Should(HaveKey("Settings"))
+
+				// only get one item
+				settings := value["Settings"].(map[string]interface{})
+				exceptionList := settings["Exceptions"].([]interface{})
+				Expect(exceptionList).Should(HaveLen(1))
+				Expect(exceptionList[0].(string)).Should(Equal("10.244.0.0/16"))
+			})
+
+			It("nothing to do if CIDR is blank", func() {
+				// mock different OutBoundNAT routes
+				n.Policies = []Policy{
+					{
+						Name:  "EndpointPolicy",
+						Value: bprintf(`{"Type": "OutBoundNAT", "Settings": {"Exceptions": []}}`),
+					},
+				}
+				n.ApplyOutboundNatPolicy("")
+
+				// only one policy
+				addlArgs := n.Policies
+				Expect(addlArgs).Should(HaveLen(1))
+
+				// normal type judgement
+				policy := addlArgs[0]
+				Expect(policy.Name).Should(Equal("EndpointPolicy"))
+				value := make(map[string]interface{})
+				json.Unmarshal(policy.Value, &value)
+				Expect(value).Should(HaveKey("Type"))
+				Expect(value["Type"]).Should(Equal("OutBoundNAT"))
+				Expect(value).Should(HaveKey("Settings"))
+
+				// empty list
+				settings := value["Settings"].(map[string]interface{})
+				Expect(settings["Exceptions"]).ShouldNot(BeNil())
+				Expect(settings["Exceptions"]).Should(HaveLen(0))
 			})
 		})
 	})
 
 	Describe("ApplyDefaultPAPolicy", func() {
-		Context("when not set by user", func() {
-			It("sets it by adding a policy", func() {
-
-				n := NetConf{}
-				n.ApplyDefaultPAPolicy("192.168.0.1")
-
-				addlArgs := n.Policies
-				Expect(addlArgs).Should(HaveLen(1))
-
-				policy := addlArgs[0]
-				Expect(policy.Name).Should(Equal("EndpointPolicy"))
-
-				value := make(map[string]interface{})
-				json.Unmarshal(policy.Value, &value)
-
-				Expect(value).Should(HaveKey("Type"))
-				Expect(value["Type"]).Should(Equal("PA"))
-
-				paAddress := value["PA"].(string)
-				Expect(paAddress).Should(Equal("192.168.0.1"))
+		Context("via v1 api", func() {
+			var n NetConf
+			BeforeEach(func() {
+				n = NetConf{}
 			})
-		})
 
-		Context("when set by user", func() {
-			It("does not override", func() {
-				n := NetConf{}
+			It("append different IP", func() {
+				// mock different IP
 				n.ApplyDefaultPAPolicy("192.168.0.1")
 				n.ApplyDefaultPAPolicy("192.168.0.2")
 
+				// will be two policies
 				addlArgs := n.Policies
-				Expect(addlArgs).Should(HaveLen(1))
+				Expect(addlArgs).Should(HaveLen(2))
 
-				policy := addlArgs[0]
+				// normal type judgement
+				policy := addlArgs[1] // judge second item
 				Expect(policy.Name).Should(Equal("EndpointPolicy"))
-
 				value := make(map[string]interface{})
 				json.Unmarshal(policy.Value, &value)
-
 				Expect(value).Should(HaveKey("Type"))
 				Expect(value["Type"]).Should(Equal("PA"))
 
+				// compare with second item
 				paAddress := value["PA"].(string)
-				Expect(paAddress).Should(Equal("192.168.0.1"))
-				Expect(paAddress).ShouldNot(Equal("192.168.0.2"))
+				Expect(paAddress).Should(Equal("192.168.0.2"))
+			})
+
+			It("nothing to do if IP is blank", func() {
+				// mock different policy
+				n.Policies = []Policy{
+					{
+						Name:  "EndpointPolicy",
+						Value: bprintf(`{"Type": "OutBoundNAT", "Exceptions": ["192.168.0.0/16"]}`),
+					},
+				}
+				n.ApplyDefaultPAPolicy("")
+
+				// nothing
+				addlArgs := n.Policies
+				Expect(addlArgs).Should(HaveLen(1))
+			})
+		})
+
+		Context("via v2 api", func() {
+			var n NetConf
+			BeforeEach(func() {
+				n = NetConf{ApiVersion: 2}
+			})
+
+			It("append different IP", func() {
+				// mock different IP
+				n.ApplyDefaultPAPolicy("192.168.0.1")
+				n.ApplyDefaultPAPolicy("192.168.0.2")
+
+				// will be two policies
+				addlArgs := n.Policies
+				Expect(addlArgs).Should(HaveLen(2))
+
+				// normal type judgement
+				policy := addlArgs[1] // judge second item
+				Expect(policy.Name).Should(Equal("EndpointPolicy"))
+				value := make(map[string]interface{})
+				json.Unmarshal(policy.Value, &value)
+				Expect(value).Should(HaveKey("Type"))
+				Expect(value["Type"]).Should(Equal("ProviderAddress"))
+				Expect(value).Should(HaveKey("Settings"))
+
+				// compare with second item
+				settings := value["Settings"].(map[string]interface{})
+				paAddress := settings["ProviderAddress"].(string)
+				Expect(paAddress).Should(Equal("192.168.0.2"))
+			})
+
+			It("nothing to do if IP is blank", func() {
+				// mock different policy
+				n.Policies = []Policy{
+					{
+						Name:  "EndpointPolicy",
+						Value: bprintf(`{"Type": "OutBoundNAT", "Settings": {"Exceptions": ["192.168.0.0/16"]}}`),
+					},
+				}
+				n.ApplyDefaultPAPolicy("")
+
+				// nothing
+				addlArgs := n.Policies
+				Expect(addlArgs).Should(HaveLen(1))
 			})
 		})
 	})
 
 	Describe("ApplyPortMappingPolicy", func() {
-		Context("when portMappings not activated", func() {
-			It("does nothing", func() {
-				n := NetConf{}
+		Context("via v1 api", func() {
+			var n NetConf
+			BeforeEach(func() {
+				n = NetConf{}
+			})
+
+			It("nothing to do if input is empty", func() {
 				n.ApplyPortMappingPolicy(nil)
 				Expect(n.Policies).Should(BeNil())
 
 				n.ApplyPortMappingPolicy([]PortMapEntry{})
-				Expect(n.Policies).Should(HaveLen(0))
+				Expect(n.Policies).Should(BeNil())
 			})
-		})
 
-		Context("when portMappings is activated", func() {
-			It("creates NAT policies", func() {
-				n := NetConf{}
+			It("create one NAT policy", func() {
+				// mock different IP
 				n.ApplyPortMappingPolicy([]PortMapEntry{
 					{
 						ContainerPort: 80,
 						HostPort:      8080,
 						Protocol:      "TCP",
-						HostIP:        "ignored",
+						HostIP:        "192.168.1.2",
 					},
 				})
 
-				Expect(n.Policies).Should(HaveLen(1))
+				// only one item
+				addlArgs := n.Policies
+				Expect(addlArgs).Should(HaveLen(1))
 
-				policy := n.Policies[0]
+				// normal type judgement
+				policy := addlArgs[0]
 				Expect(policy.Name).Should(Equal("EndpointPolicy"))
-
 				value := make(map[string]interface{})
 				json.Unmarshal(policy.Value, &value)
-
 				Expect(value).Should(HaveKey("Type"))
 				Expect(value["Type"]).Should(Equal("NAT"))
 
+				// compare all values
 				Expect(value).Should(HaveKey("InternalPort"))
 				Expect(value["InternalPort"]).Should(Equal(float64(80)))
-
 				Expect(value).Should(HaveKey("ExternalPort"))
 				Expect(value["ExternalPort"]).Should(Equal(float64(8080)))
-
 				Expect(value).Should(HaveKey("Protocol"))
 				Expect(value["Protocol"]).Should(Equal("TCP"))
 			})
 		})
-	})
 
-	Describe("MarshalPolicies", func() {
-		Context("when not set by user", func() {
-			It("sets it by adding a policy", func() {
-
-				n := NetConf{
-					Policies: []policy{
-						{
-							Name:  "EndpointPolicy",
-							Value: []byte(`{"someKey": "someValue"}`),
-						},
-						{
-							Name:  "someOtherType",
-							Value: []byte(`{"someOtherKey": "someOtherValue"}`),
-						},
-					},
-				}
-
-				result := n.MarshalPolicies()
-				Expect(len(result)).To(Equal(1))
-
-				policy := make(map[string]interface{})
-				err := json.Unmarshal(result[0], &policy)
-				Expect(err).ToNot(HaveOccurred())
-				Expect(policy).Should(HaveKey("someKey"))
-				Expect(policy["someKey"]).To(Equal("someValue"))
+		Context("via v2 api", func() {
+			var n NetConf
+			BeforeEach(func() {
+				n = NetConf{ApiVersion: 2}
 			})
-		})
 
-		Context("when set by user", func() {
-			It("appends exceptions to the existing policy", func() {
-				// first set it
-				n := NetConf{}
-				n.ApplyOutboundNatPolicy("192.168.0.0/16")
+			It("nothing to do if input is empty", func() {
+				n.ApplyPortMappingPolicy(nil)
+				Expect(n.Policies).Should(BeNil())
 
-				// then attempt to update it
-				n.ApplyOutboundNatPolicy("10.244.0.0/16")
+				n.ApplyPortMappingPolicy([]PortMapEntry{})
+				Expect(n.Policies).Should(BeNil())
+			})
 
-				// it should be unchanged!
+			It("creates one NAT policy", func() {
+				// mock different IP
+				n.ApplyPortMappingPolicy([]PortMapEntry{
+					{
+						ContainerPort: 80,
+						HostPort:      8080,
+						Protocol:      "TCP",
+						HostIP:        "192.168.1.2",
+					},
+				})
+
+				// only one item
 				addlArgs := n.Policies
 				Expect(addlArgs).Should(HaveLen(1))
 
+				// normal type judgement
 				policy := addlArgs[0]
 				Expect(policy.Name).Should(Equal("EndpointPolicy"))
-
-				var value map[string]interface{}
+				value := make(map[string]interface{})
 				json.Unmarshal(policy.Value, &value)
-
 				Expect(value).Should(HaveKey("Type"))
-				Expect(value).Should(HaveKey("ExceptionList"))
-				Expect(value["Type"]).Should(Equal("OutBoundNAT"))
+				Expect(value["Type"]).Should(Equal("PortMapping"))
+				Expect(value).Should(HaveKey("Settings"))
 
-				exceptionList := value["ExceptionList"].([]interface{})
-				Expect(exceptionList).Should(HaveLen(2))
-				Expect(exceptionList[0].(string)).Should(Equal("192.168.0.0/16"))
-				Expect(exceptionList[1].(string)).Should(Equal("10.244.0.0/16"))
+				// compare all values
+				settings := value["Settings"].(map[string]interface{})
+				Expect(settings).Should(HaveKey("InternalPort"))
+				Expect(settings["InternalPort"]).Should(Equal(float64(80)))
+				Expect(settings).Should(HaveKey("ExternalPort"))
+				Expect(settings["ExternalPort"]).Should(Equal(float64(8080)))
+				Expect(settings).Should(HaveKey("Protocol"))
+				Expect(settings["Protocol"]).Should(Equal(float64(6)))
+				Expect(settings).Should(HaveKey("VIP"))
+				Expect(settings["VIP"]).Should(Equal("192.168.1.2"))
+			})
+		})
+	})
+
+	Describe("GetXEndpointPolicies", func() {
+		Context("via v1 api", func() {
+			var n NetConf
+			BeforeEach(func() {
+				n = NetConf{}
+			})
+
+			It("GetHNSEndpointPolicies", func() {
+				// mock different policies
+				n.Policies = []Policy{
+					{
+						Name:  "EndpointPolicy",
+						Value: []byte(`{"Type": "OutBoundNAT", "ExceptionList": [ "192.168.1.2" ]}`),
+					},
+					{
+						Name:  "someOtherType",
+						Value: []byte(`{"someOtherKey": "someOtherValue"}`),
+					},
+				}
+
+				// only one valid item
+				result := n.GetHNSEndpointPolicies()
+				Expect(len(result)).To(Equal(1))
+
+				// normal type judgement
+				policy := make(map[string]interface{})
+				err := json.Unmarshal(result[0], &policy)
+				Expect(err).ToNot(HaveOccurred())
+				Expect(policy).Should(HaveKey("Type"))
+				Expect(policy["Type"]).To(Equal("OutBoundNAT"))
+				Expect(policy).Should(HaveKey("ExceptionList"))
+				Expect(policy["ExceptionList"]).To(ContainElement("192.168.1.2"))
+			})
+		})
+
+		Context("via v2 api", func() {
+			var n NetConf
+			BeforeEach(func() {
+				n = NetConf{ApiVersion: 2}
+			})
+
+			It("GetHostComputeEndpointPolicies", func() {
+				// mock different policies
+				n.Policies = []Policy{
+					{
+						Name:  "EndpointPolicy",
+						Value: []byte(`{"Type": "OutBoundNAT", "Settings": {"Exceptions": [ "192.168.1.2" ]}}`),
+					},
+					{
+						Name:  "someOtherType",
+						Value: []byte(`{"someOtherKey": "someOtherValue"}`),
+					},
+				}
+
+				// only one valid item
+				result := n.GetHostComputeEndpointPolicies()
+				Expect(len(result)).To(Equal(1))
+
+				// normal type judgement
+				policy := result[0]
+				Expect(policy.Type).Should(Equal(hcn.OutBoundNAT))
+				settings := make(map[string]interface{})
+				err := json.Unmarshal(policy.Settings, &settings)
+				Expect(err).ToNot(HaveOccurred())
+				Expect(settings["Exceptions"]).To(ContainElement("192.168.1.2"))
 			})
 		})
 	})

pkg/ip/ip.go

@@ -0,0 +1,105 @@
+// Copyright 2021 CNI authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package ip
+
+import (
+	"fmt"
+	"net"
+	"strings"
+)
+
+// IP is a CNI maintained type inherited from net.IPNet which can
+// represent a single IP address with or without prefix.
+type IP struct {
+	net.IPNet
+}
+
+// newIP will create an IP with net.IP and net.IPMask
+func newIP(ip net.IP, mask net.IPMask) *IP {
+	return &IP{
+		IPNet: net.IPNet{
+			IP:   ip,
+			Mask: mask,
+		},
+	}
+}
+
+// ParseIP will parse string s as an IP, and return it.
+// The string s must be formed like <ip>[/<prefix>].
+// If s is not a valid textual representation of an IP,
+// will return nil.
+func ParseIP(s string) *IP {
+	if strings.ContainsAny(s, "/") {
+		ip, ipNet, err := net.ParseCIDR(s)
+		if err != nil {
+			return nil
+		}
+		return newIP(ip, ipNet.Mask)
+	} else {
+		ip := net.ParseIP(s)
+		if ip == nil {
+			return nil
+		}
+		return newIP(ip, nil)
+	}
+}
+
+// ToIP will return a net.IP in standard form from this IP.
+// If this IP can not be converted to a valid net.IP, will return nil.
+func (i *IP) ToIP() net.IP {
+	switch {
+	case i.IP.To4() != nil:
+		return i.IP.To4()
+	case i.IP.To16() != nil:
+		return i.IP.To16()
+	default:
+		return nil
+	}
+}
+
+// String returns the string form of this IP.
+func (i *IP) String() string {
+	if len(i.Mask) > 0 {
+		return i.IPNet.String()
+	}
+	return i.IP.String()
+}
+
+// MarshalText implements the encoding.TextMarshaler interface.
+// The encoding is the same as returned by String,
+// But when len(ip) is zero, will return an empty slice.
+func (i *IP) MarshalText() ([]byte, error) {
+	if len(i.IP) == 0 {
+		return []byte{}, nil
+	}
+	return []byte(i.String()), nil
+}
+
+// UnmarshalText implements the encoding.TextUnmarshaler interface.
+// The textual bytes are expected in a form accepted by Parse,
+// But when len(b) is zero, will return an empty IP.
+func (i *IP) UnmarshalText(b []byte) error {
+	if len(b) == 0 {
+		*i = IP{}
+		return nil
+	}
+
+	ip := ParseIP(string(b))
+	if ip == nil {
+		return fmt.Errorf("invalid IP address %s", string(b))
+	}
+	*i = *ip
+	return nil
+}

pkg/ip/ip_test.go

@@ -0,0 +1,272 @@
+// Copyright 2021 CNI authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package ip
+
+import (
+	"encoding/json"
+	"fmt"
+	"net"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("IP Operations", func() {
+	It("Parse", func() {
+		testCases := []struct {
+			ipStr    string
+			expected *IP
+		}{
+			{
+				"192.168.0.10",
+				newIP(net.IPv4(192, 168, 0, 10), nil),
+			},
+			{
+				"2001:db8::1",
+				newIP(net.ParseIP("2001:db8::1"), nil),
+			},
+			{
+				"192.168.0.10/24",
+				newIP(net.IPv4(192, 168, 0, 10), net.IPv4Mask(255, 255, 255, 0)),
+			},
+			{
+				"2001:db8::1/64",
+				newIP(net.ParseIP("2001:db8::1"), net.CIDRMask(64, 128)),
+			},
+			{
+				"invalid",
+				nil,
+			},
+		}
+
+		for _, test := range testCases {
+			ip := ParseIP(test.ipStr)
+
+			Expect(ip).To(Equal(test.expected))
+		}
+	})
+
+	It("String", func() {
+		testCases := []struct {
+			ip       *IP
+			expected string
+		}{
+			{
+				newIP(net.IPv4(192, 168, 0, 1), net.IPv4Mask(255, 255, 255, 0)),
+				"192.168.0.1/24",
+			},
+			{
+				newIP(net.IPv4(192, 168, 0, 2), nil),
+				"192.168.0.2",
+			},
+			{
+				newIP(net.ParseIP("2001:db8::1"), nil),
+				"2001:db8::1",
+			},
+			{
+				newIP(net.ParseIP("2001:db8::1"), net.CIDRMask(64, 128)),
+				"2001:db8::1/64",
+			},
+			{
+				newIP(nil, nil),
+				"<nil>",
+			},
+		}
+
+		for _, test := range testCases {
+			Expect(test.ip.String()).To(Equal(test.expected))
+		}
+	})
+
+	It("ToIP", func() {
+		testCases := []struct {
+			ip          *IP
+			expectedLen int
+			expectedIP  net.IP
+		}{
+			{
+				newIP(net.IPv4(192, 168, 0, 1), net.IPv4Mask(255, 255, 255, 0)),
+				net.IPv4len,
+				net.IP{192, 168, 0, 1},
+			},
+			{
+				newIP(net.IPv4(192, 168, 0, 2), nil),
+				net.IPv4len,
+				net.IP{192, 168, 0, 2},
+			},
+			{
+				newIP(net.ParseIP("2001:db8::1"), nil),
+				net.IPv6len,
+				net.IP{32, 1, 13, 184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1},
+			},
+			{
+				newIP(net.ParseIP("2001:db8::1"), net.CIDRMask(64, 128)),
+				net.IPv6len,
+				net.IP{32, 1, 13, 184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1},
+			},
+			{
+				newIP(nil, nil),
+				0,
+				nil,
+			},
+		}
+
+		for _, test := range testCases {
+			Expect(len(test.ip.ToIP())).To(Equal(test.expectedLen))
+			Expect(test.ip.ToIP()).To(Equal(test.expectedIP))
+		}
+	})
+
+	It("Encode", func() {
+		testCases := []struct {
+			object   interface{}
+			expected string
+		}{
+			{
+				newIP(net.IPv4(192, 168, 0, 1), net.IPv4Mask(255, 255, 255, 0)),
+				`"192.168.0.1/24"`,
+			},
+			{
+				newIP(net.IPv4(192, 168, 0, 2), nil),
+				`"192.168.0.2"`,
+			},
+			{
+				newIP(net.ParseIP("2001:db8::1"), nil),
+				`"2001:db8::1"`,
+			},
+			{
+				newIP(net.ParseIP("2001:db8::1"), net.CIDRMask(64, 128)),
+				`"2001:db8::1/64"`,
+			},
+			{
+				newIP(nil, nil),
+				`""`,
+			},
+			{
+				[]*IP{
+					newIP(net.IPv4(192, 168, 0, 1), net.IPv4Mask(255, 255, 255, 0)),
+					newIP(net.IPv4(192, 168, 0, 2), nil),
+					newIP(net.ParseIP("2001:db8::1"), nil),
+					newIP(net.ParseIP("2001:db8::1"), net.CIDRMask(64, 128)),
+					newIP(nil, nil),
+				},
+				`["192.168.0.1/24","192.168.0.2","2001:db8::1","2001:db8::1/64",""]`,
+			},
+		}
+
+		for _, test := range testCases {
+			bytes, err := json.Marshal(test.object)
+
+			Expect(err).NotTo(HaveOccurred())
+			Expect(string(bytes)).To(Equal(test.expected))
+		}
+	})
+
+	It("Decode", func() {
+		Context("valid IP", func() {
+			testCases := []struct {
+				text     string
+				expected *IP
+			}{
+				{
+					`"192.168.0.1"`,
+					newIP(net.IPv4(192, 168, 0, 1), nil),
+				},
+				{
+					`"192.168.0.1/24"`,
+					newIP(net.IPv4(192, 168, 0, 1), net.IPv4Mask(255, 255, 255, 0)),
+				},
+				{
+					`"2001:db8::1"`,
+					newIP(net.ParseIP("2001:db8::1"), nil),
+				},
+				{
+					`"2001:db8::1/64"`,
+					newIP(net.ParseIP("2001:db8::1"), net.CIDRMask(64, 128)),
+				},
+			}
+
+			for _, test := range testCases {
+				ip := &IP{}
+				err := json.Unmarshal([]byte(test.text), ip)
+
+				Expect(err).NotTo(HaveOccurred())
+				Expect(ip).To(Equal(test.expected))
+			}
+
+		})
+
+		Context("empty text", func() {
+			ip := &IP{}
+			err := json.Unmarshal([]byte(`""`), ip)
+
+			Expect(err).NotTo(HaveOccurred())
+			Expect(ip).To(Equal(newIP(nil, nil)))
+		})
+
+		Context("invalid IP", func() {
+			testCases := []struct {
+				text        string
+				expectedErr error
+			}{
+				{
+					`"192.168.0.1000"`,
+					fmt.Errorf("invalid IP address 192.168.0.1000"),
+				},
+				{
+					`"2001:db8::1/256"`,
+					fmt.Errorf("invalid IP address 2001:db8::1/256"),
+				},
+				{
+					`"test"`,
+					fmt.Errorf("invalid IP address test"),
+				},
+			}
+
+			for _, test := range testCases {
+				err := json.Unmarshal([]byte(test.text), &IP{})
+
+				Expect(err).To(HaveOccurred())
+				Expect(err).To(Equal(test.expectedErr))
+			}
+		})
+
+		Context("IP slice", func() {
+			testCases := []struct {
+				text     string
+				expected []*IP
+			}{
+				{
+					`["192.168.0.1/24","192.168.0.2","2001:db8::1","2001:db8::1/64",""]`,
+					[]*IP{
+						newIP(net.IPv4(192, 168, 0, 1), net.IPv4Mask(255, 255, 255, 0)),
+						newIP(net.IPv4(192, 168, 0, 2), nil),
+						newIP(net.ParseIP("2001:db8::1"), nil),
+						newIP(net.ParseIP("2001:db8::1"), net.CIDRMask(64, 128)),
+						newIP(nil, nil),
+					},
+				},
+			}
+
+			for _, test := range testCases {
+				ips := make([]*IP, 0)
+				err := json.Unmarshal([]byte(test.text), &ips)
+
+				Expect(err).NotTo(HaveOccurred())
+				Expect(ips).To(Equal(test.expected))
+			}
+		})
+	})
+})

pkg/ip/ipforward_linux.go

@@ -18,7 +18,7 @@ import (
 	"bytes"
 	"io/ioutil"
 
-	"github.com/containernetworking/cni/pkg/types/current"
+	current "github.com/containernetworking/cni/pkg/types/100"
 )
 
 func EnableIP4Forward() error {
@@ -36,12 +36,13 @@ func EnableForward(ips []*current.IPConfig) error {
 	v6 := false
 
 	for _, ip := range ips {
-		if ip.Version == "4" && !v4 {
+		isV4 := ip.Address.IP.To4() != nil
+		if isV4 && !v4 {
 			if err := EnableIP4Forward(); err != nil {
 				return err
 			}
 			v4 = true
-		} else if ip.Version == "6" && !v6 {
+		} else if !isV4 && !v6 {
 			if err := EnableIP6Forward(); err != nil {
 				return err
 			}

pkg/ip/link_linux.go

@@ -25,7 +25,6 @@ import (
 	"github.com/vishvananda/netlink"
 
 	"github.com/containernetworking/plugins/pkg/ns"
-	"github.com/containernetworking/plugins/pkg/utils/hwaddr"
 	"github.com/containernetworking/plugins/pkg/utils/sysctl"
 )
 
@@ -33,19 +32,27 @@ var (
 	ErrLinkNotFound = errors.New("link not found")
 )
 
-func makeVethPair(name, peer string, mtu int) (netlink.Link, error) {
+// makeVethPair is called from within the container's network namespace
+func makeVethPair(name, peer string, mtu int, mac string, hostNS ns.NetNS) (netlink.Link, error) {
 	veth := &netlink.Veth{
 		LinkAttrs: netlink.LinkAttrs{
-			Name:  name,
-			Flags: net.FlagUp,
-			MTU:   mtu,
+			Name: name,
+			MTU:  mtu,
 		},
-		PeerName: peer,
+		PeerName:      peer,
+		PeerNamespace: netlink.NsFd(int(hostNS.Fd())),
+	}
+	if mac != "" {
+		m, err := net.ParseMAC(mac)
+		if err != nil {
+			return nil, err
+		}
+		veth.LinkAttrs.HardwareAddr = m
 	}
 	if err := netlink.LinkAdd(veth); err != nil {
 		return nil, err
 	}
-	// Re-fetch the link to get its creation-time parameters, e.g. index and mac
+	// Re-fetch the container link to get its creation-time parameters, e.g. index and mac
 	veth2, err := netlink.LinkByName(name)
 	if err != nil {
 		netlink.LinkDel(veth) // try and clean up the link if possible.
@@ -62,7 +69,7 @@ func peerExists(name string) bool {
 	return true
 }
 
-func makeVeth(name, vethPeerName string, mtu int) (peerName string, veth netlink.Link, err error) {
+func makeVeth(name, vethPeerName string, mtu int, mac string, hostNS ns.NetNS) (peerName string, veth netlink.Link, err error) {
 	for i := 0; i < 10; i++ {
 		if vethPeerName != "" {
 			peerName = vethPeerName
@@ -73,7 +80,7 @@ func makeVeth(name, vethPeerName string, mtu int) (peerName string, veth netlink
 			}
 		}
 
-		veth, err = makeVethPair(name, peerName, mtu)
+		veth, err = makeVethPair(name, peerName, mtu, mac, hostNS)
 		switch {
 		case err == nil:
 			return
@@ -99,7 +106,7 @@ func makeVeth(name, vethPeerName string, mtu int) (peerName string, veth netlink
 // RandomVethName returns string "veth" with random prefix (hashed from entropy)
 func RandomVethName() (string, error) {
 	entropy := make([]byte, 4)
-	_, err := rand.Reader.Read(entropy)
+	_, err := rand.Read(entropy)
 	if err != nil {
 		return "", fmt.Errorf("failed to generate random veth name: %v", err)
 	}
@@ -132,25 +139,13 @@ func ifaceFromNetlinkLink(l netlink.Link) net.Interface {
 // devices and move the host-side veth into the provided hostNS namespace.
 // hostVethName: If hostVethName is not specified, the host-side veth name will use a random string.
 // On success, SetupVethWithName returns (hostVeth, containerVeth, nil)
-func SetupVethWithName(contVethName, hostVethName string, mtu int, hostNS ns.NetNS) (net.Interface, net.Interface, error) {
-	hostVethName, contVeth, err := makeVeth(contVethName, hostVethName, mtu)
+func SetupVethWithName(contVethName, hostVethName string, mtu int, contVethMac string, hostNS ns.NetNS) (net.Interface, net.Interface, error) {
+	hostVethName, contVeth, err := makeVeth(contVethName, hostVethName, mtu, contVethMac, hostNS)
 	if err != nil {
 		return net.Interface{}, net.Interface{}, err
 	}
 
-	if err = netlink.LinkSetUp(contVeth); err != nil {
-		return net.Interface{}, net.Interface{}, fmt.Errorf("failed to set %q up: %v", contVethName, err)
-	}
-
-	hostVeth, err := netlink.LinkByName(hostVethName)
-	if err != nil {
-		return net.Interface{}, net.Interface{}, fmt.Errorf("failed to lookup %q: %v", hostVethName, err)
-	}
-
-	if err = netlink.LinkSetNsFd(hostVeth, int(hostNS.Fd())); err != nil {
-		return net.Interface{}, net.Interface{}, fmt.Errorf("failed to move veth to host netns: %v", err)
-	}
-
+	var hostVeth netlink.Link
 	err = hostNS.Do(func(_ ns.NetNS) error {
 		hostVeth, err = netlink.LinkByName(hostVethName)
 		if err != nil {
@@ -175,8 +170,8 @@ func SetupVethWithName(contVethName, hostVethName string, mtu int, hostNS ns.Net
 // Call SetupVeth from inside the container netns.  It will create both veth
 // devices and move the host-side veth into the provided hostNS namespace.
 // On success, SetupVeth returns (hostVeth, containerVeth, nil)
-func SetupVeth(contVethName string, mtu int, hostNS ns.NetNS) (net.Interface, net.Interface, error) {
-	return SetupVethWithName(contVethName, "", mtu, hostNS)
+func SetupVeth(contVethName string, mtu int, contVethMac string, hostNS ns.NetNS) (net.Interface, net.Interface, error) {
+	return SetupVethWithName(contVethName, "", mtu, contVethMac, hostNS)
 }
 
 // DelLinkByName removes an interface link.
@@ -225,33 +220,6 @@ func DelLinkByNameAddr(ifName string) ([]*net.IPNet, error) {
 	return out, nil
 }
 
-func SetHWAddrByIP(ifName string, ip4 net.IP, ip6 net.IP) error {
-	iface, err := netlink.LinkByName(ifName)
-	if err != nil {
-		return fmt.Errorf("failed to lookup %q: %v", ifName, err)
-	}
-
-	switch {
-	case ip4 == nil && ip6 == nil:
-		return fmt.Errorf("neither ip4 or ip6 specified")
-
-	case ip4 != nil:
-		{
-			hwAddr, err := hwaddr.GenerateHardwareAddr4(ip4, hwaddr.PrivateMACPrefix)
-			if err != nil {
-				return fmt.Errorf("failed to generate hardware addr: %v", err)
-			}
-			if err = netlink.LinkSetHardwareAddr(iface, hwAddr); err != nil {
-				return fmt.Errorf("failed to add hardware addr to %q: %v", ifName, err)
-			}
-		}
-	case ip6 != nil:
-		// TODO: IPv6
-	}
-
-	return nil
-}
-
 // GetVethPeerIfindex returns the veth link object, the peer ifindex of the
 // veth, or an error. This peer ifindex will only be valid in the peer's
 // network namespace.

pkg/ip/link_linux_test.go

@@ -51,8 +51,6 @@ var _ = Describe("Link", func() {
 		hostVethName      string
 		containerVethName string
 
-		ip4one             = net.ParseIP("1.1.1.1")
-		ip4two             = net.ParseIP("1.1.1.2")
 		originalRandReader = rand.Reader
 	)
 
@@ -72,7 +70,7 @@ var _ = Describe("Link", func() {
 		_ = containerNetNS.Do(func(ns.NetNS) error {
 			defer GinkgoRecover()
 
-			hostVeth, containerVeth, err = ip.SetupVeth(fmt.Sprintf(ifaceFormatString, ifaceCounter), mtu, hostNetNS)
+			hostVeth, containerVeth, err = ip.SetupVeth(fmt.Sprintf(ifaceFormatString, ifaceCounter), mtu, "", hostNetNS)
 			if err != nil {
 				return err
 			}
@@ -159,7 +157,7 @@ var _ = Describe("Link", func() {
 			_ = containerNetNS.Do(func(ns.NetNS) error {
 				defer GinkgoRecover()
 
-				_, _, err := ip.SetupVeth(containerVethName, mtu, hostNetNS)
+				_, _, err := ip.SetupVeth(containerVethName, mtu, "", hostNetNS)
 				Expect(err.Error()).To(Equal(fmt.Sprintf("container veth name provided (%s) already exists", containerVethName)))
 
 				return nil
@@ -189,9 +187,9 @@ var _ = Describe("Link", func() {
 		It("returns useful error", func() {
 			_ = containerNetNS.Do(func(ns.NetNS) error {
 				defer GinkgoRecover()
-				_, _, err := ip.SetupVeth(containerVethName, mtu, hostNetNS)
-				Expect(err.Error()).To(HavePrefix("failed to move veth to host netns: "))
-
+				_, _, err := ip.SetupVeth(containerVethName, mtu, "", hostNetNS)
+				Expect(err.Error()).To(HavePrefix("container veth name provided"))
+				Expect(err.Error()).To(HaveSuffix("already exists"))
 				return nil
 			})
 		})
@@ -207,7 +205,7 @@ var _ = Describe("Link", func() {
 			_ = containerNetNS.Do(func(ns.NetNS) error {
 				defer GinkgoRecover()
 
-				hostVeth, _, err := ip.SetupVeth(containerVethName, mtu, hostNetNS)
+				hostVeth, _, err := ip.SetupVeth(containerVethName, mtu, "", hostNetNS)
 				Expect(err).NotTo(HaveOccurred())
 				hostVethName = hostVeth.Name
 				return nil
@@ -233,6 +231,32 @@ var _ = Describe("Link", func() {
 			})
 		})
 
+		It("successfully creates a veth pair with an explicit mac", func() {
+			const mac = "02:00:00:00:01:23"
+			_ = containerNetNS.Do(func(ns.NetNS) error {
+				defer GinkgoRecover()
+
+				hostVeth, _, err := ip.SetupVeth(containerVethName, mtu, mac, hostNetNS)
+				Expect(err).NotTo(HaveOccurred())
+				hostVethName = hostVeth.Name
+
+				link, err := netlink.LinkByName(containerVethName)
+				Expect(err).NotTo(HaveOccurred())
+				Expect(link.Attrs().HardwareAddr.String()).To(Equal(mac))
+
+				return nil
+			})
+
+			_ = hostNetNS.Do(func(ns.NetNS) error {
+				defer GinkgoRecover()
+
+				link, err := netlink.LinkByName(hostVethName)
+				Expect(err).NotTo(HaveOccurred())
+				Expect(link.Attrs().HardwareAddr.String()).NotTo(Equal(mac))
+
+				return nil
+			})
+		})
 	})
 
 	It("DelLinkByName must delete the veth endpoints", func() {
@@ -270,41 +294,4 @@ var _ = Describe("Link", func() {
 			return nil
 		})
 	})
-
-	It("SetHWAddrByIP must change the interface hwaddr and be predictable", func() {
-
-		_ = containerNetNS.Do(func(ns.NetNS) error {
-			defer GinkgoRecover()
-
-			var err error
-			hwaddrBefore := getHwAddr(containerVethName)
-
-			err = ip.SetHWAddrByIP(containerVethName, ip4one, nil)
-			Expect(err).NotTo(HaveOccurred())
-			hwaddrAfter1 := getHwAddr(containerVethName)
-
-			Expect(hwaddrBefore).NotTo(Equal(hwaddrAfter1))
-			Expect(hwaddrAfter1).To(Equal(ip4onehwaddr))
-
-			return nil
-		})
-	})
-
-	It("SetHWAddrByIP must be injective", func() {
-
-		_ = containerNetNS.Do(func(ns.NetNS) error {
-			defer GinkgoRecover()
-
-			err := ip.SetHWAddrByIP(containerVethName, ip4one, nil)
-			Expect(err).NotTo(HaveOccurred())
-			hwaddrAfter1 := getHwAddr(containerVethName)
-
-			err = ip.SetHWAddrByIP(containerVethName, ip4two, nil)
-			Expect(err).NotTo(HaveOccurred())
-			hwaddrAfter2 := getHwAddr(containerVethName)
-
-			Expect(hwaddrAfter1).NotTo(Equal(hwaddrAfter2))
-			return nil
-		})
-	})
 })

pkg/ip/utils_linux.go

@@ -1,3 +1,4 @@
+//go:build linux
 // +build linux
 
 // Copyright 2016 CNI authors
@@ -21,7 +22,7 @@ import (
 	"net"
 
 	"github.com/containernetworking/cni/pkg/types"
-	"github.com/containernetworking/cni/pkg/types/current"
+	current "github.com/containernetworking/cni/pkg/types/100"
 	"github.com/vishvananda/netlink"
 )
 
@@ -57,15 +58,10 @@ func ValidateExpectedInterfaceIPs(ifName string, resultIPs []*current.IPConfig)
 
 		findGwy := &netlink.Route{Dst: ourPrefix}
 		routeFilter := netlink.RT_FILTER_DST
-		var family int
 
-		switch {
-		case ips.Version == "4":
+		family := netlink.FAMILY_V6
+		if ips.Address.IP.To4() != nil {
 			family = netlink.FAMILY_V4
-		case ips.Version == "6":
-			family = netlink.FAMILY_V6
-		default:
-			return fmt.Errorf("Invalid IP Version %v for interface %v", ips.Version, ifName)
 		}
 
 		gwy, err := netlink.RouteListFiltered(family, findGwy, routeFilter)

pkg/ipam/ipam_linux.go

@@ -19,7 +19,7 @@ import (
 	"net"
 	"os"
 
-	"github.com/containernetworking/cni/pkg/types/current"
+	current "github.com/containernetworking/cni/pkg/types/100"
 	"github.com/containernetworking/plugins/pkg/ip"
 	"github.com/containernetworking/plugins/pkg/utils/sysctl"
 
@@ -27,7 +27,8 @@ import (
 )
 
 const (
-	DisableIPv6SysctlTemplate = "net.ipv6.conf.%s.disable_ipv6"
+	// Note: use slash as separator so we can have dots in interface name (VLANs)
+	DisableIPv6SysctlTemplate = "net/ipv6/conf/%s/disable_ipv6"
 )
 
 // ConfigureIface takes the result of IPAM plugin and
@@ -42,10 +43,6 @@ func ConfigureIface(ifName string, res *current.Result) error {
 		return fmt.Errorf("failed to lookup %q: %v", ifName, err)
 	}
 
-	if err := netlink.LinkSetUp(link); err != nil {
-		return fmt.Errorf("failed to set %q UP: %v", ifName, err)
-	}
-
 	var v4gw, v6gw net.IP
 	var has_enabled_ipv6 bool = false
 	for _, ipc := range res.IPs {
@@ -60,7 +57,7 @@ func ConfigureIface(ifName string, res *current.Result) error {
 
 		// Make sure sysctl "disable_ipv6" is 0 if we are about to add
 		// an IPv6 address to the interface
-		if !has_enabled_ipv6 && ipc.Version == "6" {
+		if !has_enabled_ipv6 && ipc.Address.IP.To4() == nil {
 			// Enabled IPv6 for loopback "lo" and the interface
 			// being configured
 			for _, iface := range [2]string{"lo", ifName} {
@@ -68,8 +65,11 @@ func ConfigureIface(ifName string, res *current.Result) error {
 
 				// Read current sysctl value
 				value, err := sysctl.Sysctl(ipv6SysctlValueName)
-				if err != nil || value == "0" {
-					// FIXME: log warning if unable to read sysctl value
+				if err != nil {
+					fmt.Fprintf(os.Stderr, "ipam_linux: failed to read sysctl %q: %v\n", ipv6SysctlValueName, err)
+					continue
+				}
+				if value == "0" {
 					continue
 				}
 
@@ -95,6 +95,10 @@ func ConfigureIface(ifName string, res *current.Result) error {
 		}
 	}
 
+	if err := netlink.LinkSetUp(link); err != nil {
+		return fmt.Errorf("failed to set %q UP: %v", ifName, err)
+	}
+
 	if v6gw != nil {
 		ip.SettleAddresses(ifName, 10)
 	}
@@ -109,11 +113,14 @@ func ConfigureIface(ifName string, res *current.Result) error {
 				gw = v6gw
 			}
 		}
-		if err = ip.AddRoute(&r.Dst, gw, link); err != nil {
-			// we skip over duplicate routes as we assume the first one wins
-			if !os.IsExist(err) {
-				return fmt.Errorf("failed to add route '%v via %v dev %v': %v", r.Dst, gw, ifName, err)
-			}
+		route := netlink.Route{
+			Dst:       &r.Dst,
+			LinkIndex: link.Attrs().Index,
+			Gw:        gw,
+		}
+
+		if err = netlink.RouteAddEcmp(&route); err != nil {
+			return fmt.Errorf("failed to add route '%v via %v dev %v': %v", r.Dst, gw, ifName, err)
 		}
 	}
 

pkg/ipam/ipam_linux_test.go

@@ -19,7 +19,7 @@ import (
 	"syscall"
 
 	"github.com/containernetworking/cni/pkg/types"
-	"github.com/containernetworking/cni/pkg/types/current"
+	current "github.com/containernetworking/cni/pkg/types/100"
 	"github.com/containernetworking/plugins/pkg/ns"
 	"github.com/containernetworking/plugins/pkg/testutils"
 
@@ -109,13 +109,11 @@ var _ = Describe("ConfigureIface", func() {
 			},
 			IPs: []*current.IPConfig{
 				{
-					Version:   "4",
 					Interface: current.Int(0),
 					Address:   *ipv4,
 					Gateway:   ipgw4,
 				},
 				{
-					Version:   "6",
 					Interface: current.Int(0),
 					Address:   *ipv6,
 					Gateway:   ipgw6,
@@ -281,12 +279,10 @@ var _ = Describe("ConfigureIface", func() {
 			},
 			IPs: []*current.IPConfig{
 				{
-					Version: "4",
 					Address: *ipv4,
 					Gateway: ipgw4,
 				},
 				{
-					Version: "6",
 					Address: *ipv6,
 					Gateway: ipgw6,
 				},

pkg/link/link_suite_test.go

@@ -1,4 +1,4 @@
-// Copyright 2016 CNI authors
+// Copyright 2021 CNI authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,16 +12,16 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package hwaddr_test
+package link_test
 
 import (
+	"testing"
+
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
-
-	"testing"
 )
 
-func TestHwaddr(t *testing.T) {
+func TestIp(t *testing.T) {
 	RegisterFailHandler(Fail)
-	RunSpecs(t, "pkg/utils/hwaddr")
+	RunSpecs(t, "pkg/link")
 }

pkg/link/spoofcheck.go

@@ -0,0 +1,245 @@
+// Copyright 2021 CNI authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package link
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/networkplumbing/go-nft/nft"
+	"github.com/networkplumbing/go-nft/nft/schema"
+)
+
+const (
+	natTableName            = "nat"
+	preRoutingBaseChainName = "PREROUTING"
+)
+
+type NftConfigurer interface {
+	Apply(*nft.Config) error
+	Read() (*nft.Config, error)
+}
+
+type SpoofChecker struct {
+	iface      string
+	macAddress string
+	refID      string
+	configurer NftConfigurer
+}
+
+type defaultNftConfigurer struct{}
+
+func (_ defaultNftConfigurer) Apply(cfg *nft.Config) error {
+	return nft.ApplyConfig(cfg)
+}
+
+func (_ defaultNftConfigurer) Read() (*nft.Config, error) {
+	return nft.ReadConfig()
+}
+
+func NewSpoofChecker(iface, macAddress, refID string) *SpoofChecker {
+	return NewSpoofCheckerWithConfigurer(iface, macAddress, refID, defaultNftConfigurer{})
+}
+
+func NewSpoofCheckerWithConfigurer(iface, macAddress, refID string, configurer NftConfigurer) *SpoofChecker {
+	return &SpoofChecker{iface, macAddress, refID, configurer}
+}
+
+// Setup applies nftables configuration to restrict traffic
+// from the provided interface. Only traffic with the mentioned mac address
+// is allowed to pass, all others are blocked.
+// The configuration follows the format libvirt and ebtables implemented, allowing
+// extensions to the rules in the future.
+// refID is used to label the rules with a unique comment, identifying the rule-set.
+//
+// In order to take advantage of the nftables configuration change atomicity, the
+// following steps are taken to apply the configuration:
+// - Declare the table and chains (they will be created in case not present).
+// - Apply the rules, while first flushing the iface/mac specific regular chain rules.
+// Two transactions are used because the flush succeeds only if the table/chain it targets
+// exists. This avoids the need to query the existing state and acting upon it (a raceful pattern).
+// Although two transactions are taken place, only the 2nd one where the rules
+// are added has a real impact on the system.
+func (sc *SpoofChecker) Setup() error {
+	baseConfig := nft.NewConfig()
+
+	baseConfig.AddTable(&schema.Table{Family: schema.FamilyBridge, Name: natTableName})
+
+	baseConfig.AddChain(sc.baseChain())
+	ifaceChain := sc.ifaceChain()
+	baseConfig.AddChain(ifaceChain)
+	macChain := sc.macChain(ifaceChain.Name)
+	baseConfig.AddChain(macChain)
+
+	if err := sc.configurer.Apply(baseConfig); err != nil {
+		return fmt.Errorf("failed to setup spoof-check: %v", err)
+	}
+
+	rulesConfig := nft.NewConfig()
+
+	rulesConfig.FlushChain(ifaceChain)
+	rulesConfig.FlushChain(macChain)
+
+	rulesConfig.AddRule(sc.matchIfaceJumpToChainRule(preRoutingBaseChainName, ifaceChain.Name))
+	rulesConfig.AddRule(sc.jumpToChainRule(ifaceChain.Name, macChain.Name))
+	rulesConfig.AddRule(sc.matchMacRule(macChain.Name))
+	rulesConfig.AddRule(sc.dropRule(macChain.Name))
+
+	if err := sc.configurer.Apply(rulesConfig); err != nil {
+		return fmt.Errorf("failed to setup spoof-check: %v", err)
+	}
+
+	return nil
+}
+
+// Teardown removes the interface and mac-address specific chains and their rules.
+// The table and base-chain are expected to survive while the base-chain rule that matches the
+// interface is removed.
+func (sc *SpoofChecker) Teardown() error {
+	ifaceChain := sc.ifaceChain()
+	currentConfig, ifaceMatchRuleErr := sc.configurer.Read()
+	if ifaceMatchRuleErr == nil {
+		expectedRuleToFind := sc.matchIfaceJumpToChainRule(preRoutingBaseChainName, ifaceChain.Name)
+		// It is safer to exclude the statement matching, avoiding cases where a current statement includes
+		// additional default entries (e.g. counters).
+		ruleToFindExcludingStatements := *expectedRuleToFind
+		ruleToFindExcludingStatements.Expr = nil
+		rules := currentConfig.LookupRule(&ruleToFindExcludingStatements)
+		if len(rules) > 0 {
+			c := nft.NewConfig()
+			for _, rule := range rules {
+				c.DeleteRule(rule)
+			}
+			if err := sc.configurer.Apply(c); err != nil {
+				ifaceMatchRuleErr = fmt.Errorf("failed to delete iface match rule: %v", err)
+			}
+		} else {
+			fmt.Fprintf(os.Stderr, "spoofcheck/teardown: unable to detect iface match rule for deletion: %+v", expectedRuleToFind)
+		}
+	}
+
+	regularChainsConfig := nft.NewConfig()
+	regularChainsConfig.DeleteChain(ifaceChain)
+	regularChainsConfig.DeleteChain(sc.macChain(ifaceChain.Name))
+
+	var regularChainsErr error
+	if err := sc.configurer.Apply(regularChainsConfig); err != nil {
+		regularChainsErr = fmt.Errorf("failed to delete regular chains: %v", err)
+	}
+
+	if ifaceMatchRuleErr != nil || regularChainsErr != nil {
+		return fmt.Errorf("failed to teardown spoof-check: %v, %v", ifaceMatchRuleErr, regularChainsErr)
+	}
+	return nil
+}
+
+func (sc *SpoofChecker) matchIfaceJumpToChainRule(chain, toChain string) *schema.Rule {
+	return &schema.Rule{
+		Family: schema.FamilyBridge,
+		Table:  natTableName,
+		Chain:  chain,
+		Expr: []schema.Statement{
+			{Match: &schema.Match{
+				Op:    schema.OperEQ,
+				Left:  schema.Expression{RowData: []byte(`{"meta":{"key":"iifname"}}`)},
+				Right: schema.Expression{String: &sc.iface},
+			}},
+			{Verdict: schema.Verdict{Jump: &schema.ToTarget{Target: toChain}}},
+		},
+		Comment: ruleComment(sc.refID),
+	}
+}
+
+func (sc *SpoofChecker) jumpToChainRule(chain, toChain string) *schema.Rule {
+	return &schema.Rule{
+		Family: schema.FamilyBridge,
+		Table:  natTableName,
+		Chain:  chain,
+		Expr: []schema.Statement{
+			{Verdict: schema.Verdict{Jump: &schema.ToTarget{Target: toChain}}},
+		},
+		Comment: ruleComment(sc.refID),
+	}
+}
+
+func (sc *SpoofChecker) matchMacRule(chain string) *schema.Rule {
+	return &schema.Rule{
+		Family: schema.FamilyBridge,
+		Table:  natTableName,
+		Chain:  chain,
+		Expr: []schema.Statement{
+			{Match: &schema.Match{
+				Op: schema.OperEQ,
+				Left: schema.Expression{Payload: &schema.Payload{
+					Protocol: schema.PayloadProtocolEther,
+					Field:    schema.PayloadFieldEtherSAddr,
+				}},
+				Right: schema.Expression{String: &sc.macAddress},
+			}},
+			{Verdict: schema.Verdict{SimpleVerdict: schema.SimpleVerdict{Return: true}}},
+		},
+		Comment: ruleComment(sc.refID),
+	}
+}
+
+func (sc *SpoofChecker) dropRule(chain string) *schema.Rule {
+	macRulesIndex := nft.NewRuleIndex()
+	return &schema.Rule{
+		Family: schema.FamilyBridge,
+		Table:  natTableName,
+		Chain:  chain,
+		Index:  macRulesIndex.Next(),
+		Expr: []schema.Statement{
+			{Verdict: schema.Verdict{SimpleVerdict: schema.SimpleVerdict{Drop: true}}},
+		},
+		Comment: ruleComment(sc.refID),
+	}
+}
+
+func (_ *SpoofChecker) baseChain() *schema.Chain {
+	chainPriority := -300
+	return &schema.Chain{
+		Family: schema.FamilyBridge,
+		Table:  natTableName,
+		Name:   preRoutingBaseChainName,
+		Type:   schema.TypeFilter,
+		Hook:   schema.HookPreRouting,
+		Prio:   &chainPriority,
+		Policy: schema.PolicyAccept,
+	}
+}
+
+func (sc *SpoofChecker) ifaceChain() *schema.Chain {
+	ifaceChainName := "cni-br-iface-" + sc.refID
+	return &schema.Chain{
+		Family: schema.FamilyBridge,
+		Table:  natTableName,
+		Name:   ifaceChainName,
+	}
+}
+
+func (_ *SpoofChecker) macChain(ifaceChainName string) *schema.Chain {
+	macChainName := ifaceChainName + "-mac"
+	return &schema.Chain{
+		Family: schema.FamilyBridge,
+		Table:  natTableName,
+		Name:   macChainName,
+	}
+}
+
+func ruleComment(id string) string {
+	const refIDPrefix = "macspoofchk-"
+	return refIDPrefix + id
+}

pkg/link/spoofcheck_test.go

@@ -0,0 +1,297 @@
+// Copyright 2021 CNI authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package link_test
+
+import (
+	"fmt"
+	"github.com/networkplumbing/go-nft/nft"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/containernetworking/plugins/pkg/link"
+)
+
+var _ = Describe("spoofcheck", func() {
+	iface := "net0"
+	mac := "02:00:00:00:12:34"
+	id := "container99-net1"
+
+	Context("setup", func() {
+		It("succeeds", func() {
+			c := configurerStub{}
+			sc := link.NewSpoofCheckerWithConfigurer(iface, mac, id, &c)
+			Expect(sc.Setup()).To(Succeed())
+
+			assertExpectedTableAndChainsInSetupConfig(c)
+			assertExpectedRulesInSetupConfig(c)
+		})
+
+		It("fails to setup config when 1st apply is unsuccessful (declare table and chains)", func() {
+			c := &configurerStub{failFirstApplyConfig: true}
+			sc := link.NewSpoofCheckerWithConfigurer(iface, mac, id, c)
+			Expect(sc.Setup()).To(MatchError("failed to setup spoof-check: " + errorFirstApplyText))
+		})
+
+		It("fails to setup config when 2nd apply is unsuccessful (flush and add the rules)", func() {
+			c := &configurerStub{failSecondApplyConfig: true}
+			sc := link.NewSpoofCheckerWithConfigurer(iface, mac, id, c)
+			Expect(sc.Setup()).To(MatchError("failed to setup spoof-check: " + errorSecondApplyText))
+		})
+	})
+
+	Context("teardown", func() {
+		It("succeeds", func() {
+			existingConfig := nft.NewConfig()
+			existingConfig.FromJSON([]byte(rowConfigWithRulesOnly()))
+			c := configurerStub{readConfig: existingConfig}
+
+			sc := link.NewSpoofCheckerWithConfigurer("", "", id, &c)
+			Expect(sc.Teardown()).To(Succeed())
+
+			assertExpectedBaseChainRuleDeletionInTeardownConfig(c)
+			assertExpectedRegularChainsDeletionInTeardownConfig(c)
+		})
+
+		It("fails, 1st apply is unsuccessful (delete iface match rule)", func() {
+			config := nft.NewConfig()
+			config.FromJSON([]byte(rowConfigWithRulesOnly()))
+			c := &configurerStub{applyConfig: []*nft.Config{config}, readConfig: config, failFirstApplyConfig: true}
+			sc := link.NewSpoofCheckerWithConfigurer("", "", id, c)
+			Expect(sc.Teardown()).To(MatchError(fmt.Sprintf(
+				"failed to teardown spoof-check: failed to delete iface match rule: %s, <nil>", errorFirstApplyText,
+			)))
+		})
+
+		It("fails, read current config is unsuccessful", func() {
+			config := nft.NewConfig()
+			config.FromJSON([]byte(rowConfigWithRulesOnly()))
+			c := &configurerStub{applyConfig: []*nft.Config{config}, readConfig: config, failReadConfig: true}
+			sc := link.NewSpoofCheckerWithConfigurer("", "", id, c)
+			Expect(sc.Teardown()).To(MatchError(fmt.Sprintf(
+				"failed to teardown spoof-check: %s, <nil>", errorReadText,
+			)))
+		})
+
+		It("fails, 2nd apply is unsuccessful (delete the regular chains)", func() {
+			config := nft.NewConfig()
+			config.FromJSON([]byte(rowConfigWithRulesOnly()))
+			c := &configurerStub{applyConfig: []*nft.Config{config}, readConfig: config, failSecondApplyConfig: true}
+			sc := link.NewSpoofCheckerWithConfigurer("", "", id, c)
+			Expect(sc.Teardown()).To(MatchError(fmt.Sprintf(
+				"failed to teardown spoof-check: <nil>, failed to delete regular chains: %s", errorSecondApplyText,
+			)))
+		})
+
+		It("fails, both applies are unsuccessful", func() {
+			config := nft.NewConfig()
+			config.FromJSON([]byte(rowConfigWithRulesOnly()))
+			c := &configurerStub{
+				applyConfig:           []*nft.Config{config},
+				readConfig:            config,
+				failFirstApplyConfig:  true,
+				failSecondApplyConfig: true,
+			}
+			sc := link.NewSpoofCheckerWithConfigurer("", "", id, c)
+			Expect(sc.Teardown()).To(MatchError(fmt.Sprintf(
+				"failed to teardown spoof-check: "+
+					"failed to delete iface match rule: %s, "+
+					"failed to delete regular chains: %s",
+				errorFirstApplyText, errorSecondApplyText,
+			)))
+		})
+	})
+})
+
+func assertExpectedRegularChainsDeletionInTeardownConfig(action configurerStub) {
+	deleteRegularChainRulesJsonConfig, err := action.applyConfig[1].ToJSON()
+	ExpectWithOffset(1, err).NotTo(HaveOccurred())
+
+	expectedDeleteRegularChainRulesJsonConfig := `
+			{"nftables": [
+				{"delete": {"chain": {
+					"family": "bridge",
+					"table": "nat",
+					"name": "cni-br-iface-container99-net1"
+				}}},
+				{"delete": {"chain": {
+					"family": "bridge",
+					"table": "nat",
+					"name": "cni-br-iface-container99-net1-mac"
+				}}}
+			]}`
+
+	ExpectWithOffset(1, string(deleteRegularChainRulesJsonConfig)).To(MatchJSON(expectedDeleteRegularChainRulesJsonConfig))
+}
+
+func assertExpectedBaseChainRuleDeletionInTeardownConfig(action configurerStub) {
+	deleteBaseChainRuleJsonConfig, err := action.applyConfig[0].ToJSON()
+	Expect(err).NotTo(HaveOccurred())
+
+	expectedDeleteIfaceMatchRuleJsonConfig := `
+            {"nftables": [
+				{"delete": {"rule": {
+					"family": "bridge",
+					"table": "nat",
+					"chain": "PREROUTING",
+					"expr": [
+						{"match": {
+							"op": "==",
+							"left": {"meta": {"key": "iifname"}},
+							"right": "net0"
+						}},
+						{"jump": {"target": "cni-br-iface-container99-net1"}}
+					],
+					"comment": "macspoofchk-container99-net1"
+				}}}
+			]}`
+	Expect(string(deleteBaseChainRuleJsonConfig)).To(MatchJSON(expectedDeleteIfaceMatchRuleJsonConfig))
+}
+
+func rowConfigWithRulesOnly() string {
+	return `
+            {"nftables":[
+                {"rule":{"family":"bridge","table":"nat","chain":"PREROUTING",
+                    "expr":[
+                        {"match":{"op":"==","left":{"meta":{"key":"iifname"}},"right":"net0"}},
+                        {"jump":{"target":"cni-br-iface-container99-net1"}}
+                    ],
+                    "comment":"macspoofchk-container99-net1"}},
+                {"rule":{"family":"bridge","table":"nat","chain":"cni-br-iface-container99-net1",
+                    "expr":[
+                        {"jump":{"target":"cni-br-iface-container99-net1-mac"}}
+                    ],
+                    "comment":"macspoofchk-container99-net1"}},
+                {"rule":{"family":"bridge","table":"nat","chain":"cni-br-iface-container99-net1-mac",
+                    "expr":[
+                        {"match":{
+                            "op":"==",
+                            "left":{"payload":{"protocol":"ether","field":"saddr"}},
+                            "right":"02:00:00:00:12:34"
+                        }},
+                        {"return":null}
+                    ],
+                    "comment":"macspoofchk-container99-net1"}},
+                {"rule":{"family":"bridge","table":"nat","chain":"cni-br-iface-container99-net1-mac",
+                    "expr":[{"drop":null}],
+                    "index":0,
+                    "comment":"macspoofchk-container99-net1"}}
+            ]}`
+}
+
+func assertExpectedTableAndChainsInSetupConfig(c configurerStub) {
+	config := c.applyConfig[0]
+	jsonConfig, err := config.ToJSON()
+	ExpectWithOffset(1, err).NotTo(HaveOccurred())
+
+	expectedConfig := `
+        {"nftables": [
+            {"table": {"family": "bridge", "name": "nat"}},
+            {"chain": {
+                "family": "bridge",
+                "table": "nat",
+                "name": "PREROUTING",
+                "type": "filter",
+                "hook": "prerouting",
+                "prio": -300,
+                "policy": "accept"
+            }},
+            {"chain": {
+                "family": "bridge",
+                "table": "nat",
+                "name": "cni-br-iface-container99-net1"
+            }},
+            {"chain": {
+                "family": "bridge",
+                "table": "nat",
+                "name": "cni-br-iface-container99-net1-mac"
+            }}
+        ]}`
+	ExpectWithOffset(1, string(jsonConfig)).To(MatchJSON(expectedConfig))
+}
+
+func assertExpectedRulesInSetupConfig(c configurerStub) {
+	config := c.applyConfig[1]
+	jsonConfig, err := config.ToJSON()
+	ExpectWithOffset(1, err).NotTo(HaveOccurred())
+
+	expectedConfig := `
+            {"nftables":[
+                {"flush":{"chain":{"family":"bridge","table":"nat","name":"cni-br-iface-container99-net1"}}},
+                {"flush":{"chain":{"family":"bridge","table":"nat","name":"cni-br-iface-container99-net1-mac"}}},
+                {"rule":{"family":"bridge","table":"nat","chain":"PREROUTING",
+                    "expr":[
+                        {"match":{"op":"==","left":{"meta":{"key":"iifname"}},"right":"net0"}},
+                        {"jump":{"target":"cni-br-iface-container99-net1"}}
+                    ],
+                    "comment":"macspoofchk-container99-net1"}},
+                {"rule":{"family":"bridge","table":"nat","chain":"cni-br-iface-container99-net1",
+                    "expr":[
+                        {"jump":{"target":"cni-br-iface-container99-net1-mac"}}
+                    ],
+                    "comment":"macspoofchk-container99-net1"}},
+                {"rule":{"family":"bridge","table":"nat","chain":"cni-br-iface-container99-net1-mac",
+                    "expr":[
+                        {"match":{
+                            "op":"==",
+                            "left":{"payload":{"protocol":"ether","field":"saddr"}},
+                            "right":"02:00:00:00:12:34"
+                        }},
+                        {"return":null}
+                    ],
+                    "comment":"macspoofchk-container99-net1"}},
+                {"rule":{"family":"bridge","table":"nat","chain":"cni-br-iface-container99-net1-mac",
+                    "expr":[{"drop":null}],
+                    "index":0,
+                    "comment":"macspoofchk-container99-net1"}}
+            ]}`
+	ExpectWithOffset(1, string(jsonConfig)).To(MatchJSON(expectedConfig))
+}
+
+const (
+	errorFirstApplyText  = "1st apply failed"
+	errorSecondApplyText = "2nd apply failed"
+	errorReadText        = "read failed"
+)
+
+type configurerStub struct {
+	applyConfig []*nft.Config
+	readConfig  *nft.Config
+
+	applyCounter int
+
+	failFirstApplyConfig  bool
+	failSecondApplyConfig bool
+	failReadConfig        bool
+}
+
+func (a *configurerStub) Apply(c *nft.Config) error {
+	a.applyCounter++
+	if a.failFirstApplyConfig && a.applyCounter == 1 {
+		return fmt.Errorf(errorFirstApplyText)
+	}
+	if a.failSecondApplyConfig && a.applyCounter == 2 {
+		return fmt.Errorf(errorSecondApplyText)
+	}
+	a.applyConfig = append(a.applyConfig, c)
+	return nil
+}
+
+func (a *configurerStub) Read() (*nft.Config, error) {
+	if a.failReadConfig {
+		return nil, fmt.Errorf(errorReadText)
+	}
+	return a.readConfig, nil
+}

pkg/ns/ns_linux.go

@@ -106,8 +106,8 @@ var _ NetNS = &netNS{}
 
 const (
 	// https://github.com/torvalds/linux/blob/master/include/uapi/linux/magic.h
-	NSFS_MAGIC   = 0x6e736673
-	PROCFS_MAGIC = 0x9fa0
+	NSFS_MAGIC   = unix.NSFS_MAGIC
+	PROCFS_MAGIC = unix.PROC_SUPER_MAGIC
 )
 
 type NSPathNotExistErr struct{ msg string }

pkg/ns/ns_linux_test.go

@@ -200,7 +200,9 @@ var _ = Describe("Linux namespace operations", func() {
 				By("comparing against the netns inode of every thread in the process")
 				for _, netnsPath := range allNetNSInCurrentProcess() {
 					netnsInode, err := getInode(netnsPath)
-					Expect(err).NotTo(HaveOccurred())
+					if !os.IsNotExist(err) {
+						Expect(err).NotTo(HaveOccurred())
+					}
 					Expect(netnsInode).NotTo(Equal(createdNetNSInode))
 				}
 			})

pkg/testutils/echo/client/client.go

@@ -0,0 +1,90 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"io"
+	"net"
+)
+
+func main() {
+	target := flag.String("target", "", "the server address")
+	payload := flag.String("message", "", "the message to send to the server")
+	protocol := flag.String("protocol", "tcp", "the protocol to use with the server [udp,tcp], default tcp")
+	flag.Parse()
+
+	if *target == "" || *payload == "" {
+		flag.Usage()
+		panic("invalid arguments")
+	}
+
+	switch *protocol {
+	case "tcp":
+		connectTCP(*target, *payload)
+	case "udp":
+		connectUDP(*target, *payload)
+	default:
+		panic("invalid protocol")
+	}
+}
+
+func connectTCP(target, payload string) {
+	conn, err := net.Dial("tcp", target)
+	if err != nil {
+		panic(fmt.Sprintf("Failed to open connection to [%s] %v", target, err))
+	}
+	defer conn.Close()
+
+	_, err = conn.Write([]byte(payload))
+	if err != nil {
+		panic("Failed to send payload")
+	}
+	_, err = conn.Write([]byte("\n"))
+	if err != nil {
+		panic("Failed to send payload")
+	}
+	buf := make([]byte, 1024)
+	for {
+		n, err := conn.Read(buf)
+		fmt.Print(string(buf[:n]))
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			panic("Failed to read from socket")
+		}
+	}
+}
+
+// UDP uses a constant source port to trigger conntrack problems
+func connectUDP(target, payload string) {
+	LocalAddr, err := net.ResolveUDPAddr("udp", ":54321")
+	if err != nil {
+		panic(fmt.Sprintf("Failed to resolve UDP local address on port 54321 %v", err))
+	}
+	RemoteAddr, err := net.ResolveUDPAddr("udp", target)
+	if err != nil {
+		panic(fmt.Sprintf("Failed to resolve UDP remote address [%s] %v", target, err))
+	}
+	conn, err := net.DialUDP("udp", LocalAddr, RemoteAddr)
+	if err != nil {
+		panic(fmt.Sprintf("Failed to open connection to [%s] %v", target, err))
+	}
+	defer conn.Close()
+
+	_, err = conn.Write([]byte(payload))
+	if err != nil {
+		panic("Failed to send payload")
+	}
+	_, err = conn.Write([]byte("\n"))
+	if err != nil {
+		panic("Failed to send payload")
+	}
+
+	buf := make([]byte, 1024)
+	n, err := conn.Read(buf)
+	if err != nil {
+		panic("Failed to read from socket")
+	}
+	fmt.Print(string(buf[:n]))
+}

pkg/testutils/echo/echo_test.go

@@ -0,0 +1,98 @@
+package main_test
+
+import (
+	"fmt"
+	"io/ioutil"
+	"net"
+	"os/exec"
+	"strings"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"github.com/onsi/gomega/gbytes"
+	"github.com/onsi/gomega/gexec"
+)
+
+var serverBinaryPath, clientBinaryPath string
+
+var _ = SynchronizedBeforeSuite(func() []byte {
+	serverBinaryPath, err := gexec.Build("github.com/containernetworking/plugins/pkg/testutils/echo/server")
+	Expect(err).NotTo(HaveOccurred())
+	clientBinaryPath, err := gexec.Build("github.com/containernetworking/plugins/pkg/testutils/echo/client")
+	Expect(err).NotTo(HaveOccurred())
+	return []byte(strings.Join([]string{serverBinaryPath, clientBinaryPath}, ","))
+}, func(data []byte) {
+	binaries := strings.Split(string(data), ",")
+	serverBinaryPath = binaries[0]
+	clientBinaryPath = binaries[1]
+})
+
+var _ = SynchronizedAfterSuite(func() {}, func() {
+	gexec.CleanupBuildArtifacts()
+})
+
+var _ = Describe("Echosvr", func() {
+	var session *gexec.Session
+	BeforeEach(func() {
+		var err error
+		cmd := exec.Command(serverBinaryPath)
+		session, err = gexec.Start(cmd, GinkgoWriter, GinkgoWriter)
+		Expect(err).NotTo(HaveOccurred())
+	})
+
+	AfterEach(func() {
+		session.Kill().Wait()
+	})
+
+	Context("Server test", func() {
+		It("starts and doesn't terminate immediately", func() {
+			Consistently(session).ShouldNot(gexec.Exit())
+		})
+
+		tryConnect := func() (net.Conn, error) {
+			programOutput := session.Out.Contents()
+			addr := strings.TrimSpace(string(programOutput))
+
+			conn, err := net.Dial("tcp", addr)
+			if err != nil {
+				return nil, err
+			}
+			return conn, err
+		}
+
+		It("prints its listening address to stdout", func() {
+			Eventually(session.Out).Should(gbytes.Say("\n"))
+			conn, err := tryConnect()
+			Expect(err).NotTo(HaveOccurred())
+			conn.Close()
+		})
+
+		It("will echo data back to us", func() {
+			Eventually(session.Out).Should(gbytes.Say("\n"))
+			conn, err := tryConnect()
+			Expect(err).NotTo(HaveOccurred())
+			defer conn.Close()
+
+			fmt.Fprintf(conn, "hello\n")
+			Expect(ioutil.ReadAll(conn)).To(Equal([]byte("hello")))
+		})
+	})
+
+	Context("Client Server Test", func() {
+		It("starts and doesn't terminate immediately", func() {
+			Consistently(session).ShouldNot(gexec.Exit())
+		})
+
+		It("connects successfully using echo client", func() {
+			Eventually(session.Out).Should(gbytes.Say("\n"))
+			serverAddress := strings.TrimSpace(string(session.Out.Contents()))
+			fmt.Println("Server address", string(serverAddress))
+
+			cmd := exec.Command(clientBinaryPath, "-target", serverAddress, "-message", "hello")
+			clientSession, err := gexec.Start(cmd, GinkgoWriter, GinkgoWriter)
+			Expect(err).NotTo(HaveOccurred())
+			Eventually(clientSession.Out).Should(gbytes.Say("hello"))
+			Eventually(clientSession).Should(gexec.Exit())
+		})
+	})
+})

pkg/testutils/echo/server/main.go

@@ -10,6 +10,7 @@ import (
 	"bufio"
 	"fmt"
 	"io"
+	"log"
 	"net"
 	"os"
 	"strings"
@@ -17,21 +18,50 @@ import (
 )
 
 func main() {
+	// Start TCP server
 	listener, err := net.Listen("tcp", ":")
 	if err != nil {
 		panic(err)
 	}
+	defer listener.Close()
+	// use the same port for UDP
 	_, port, err := net.SplitHostPort(listener.Addr().String())
 	if err != nil {
 		panic(err)
 	}
 	fmt.Printf("127.0.0.1:%s\n", port)
-	for {
-		conn, err := listener.Accept()
-		if err != nil {
-			panic(err)
+	go func() {
+		for {
+			conn, err := listener.Accept()
+			if err != nil {
+				panic(err)
+			}
+			go handleConnection(conn)
+		}
+	}()
+
+	// Start UDP server
+	addr, err := net.ResolveUDPAddr("udp", fmt.Sprintf(":%s", port))
+	if err != nil {
+		log.Fatalf("Error from net.ResolveUDPAddr(): %s", err)
+	}
+	sock, err := net.ListenUDP("udp", addr)
+	if err != nil {
+		log.Fatalf("Error from ListenUDP(): %s", err)
+	}
+	defer sock.Close()
+
+	buffer := make([]byte, 1024)
+	for {
+		n, addr, err := sock.ReadFrom(buffer)
+		if err != nil {
+			log.Fatalf("Error from ReadFrom(): %s", err)
+		}
+		sock.SetWriteDeadline(time.Now().Add(1 * time.Minute))
+		n, err = sock.WriteTo(buffer[0:n], addr)
+		if err != nil {
+			return
 		}
-		go handleConnection(conn)
 	}
 }
 
@@ -53,5 +83,4 @@ func handleConnection(conn net.Conn) {
 		fmt.Fprint(os.Stderr, err.Error())
 		return
 	}
-
 }

pkg/testutils/echosvr/echosvr_test.go

@@ -1,74 +0,0 @@
-package main_test
-
-import (
-	"fmt"
-	"io/ioutil"
-	"net"
-	"os/exec"
-	"strings"
-
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-	"github.com/onsi/gomega/gbytes"
-	"github.com/onsi/gomega/gexec"
-)
-
-var binaryPath string
-
-var _ = SynchronizedBeforeSuite(func() []byte {
-	binaryPath, err := gexec.Build("github.com/containernetworking/plugins/pkg/testutils/echosvr")
-	Expect(err).NotTo(HaveOccurred())
-	return []byte(binaryPath)
-}, func(data []byte) {
-	binaryPath = string(data)
-})
-
-var _ = SynchronizedAfterSuite(func() {}, func() {
-	gexec.CleanupBuildArtifacts()
-})
-
-var _ = Describe("Echosvr", func() {
-	var session *gexec.Session
-	BeforeEach(func() {
-		var err error
-		cmd := exec.Command(binaryPath)
-		session, err = gexec.Start(cmd, GinkgoWriter, GinkgoWriter)
-		Expect(err).NotTo(HaveOccurred())
-	})
-
-	AfterEach(func() {
-		session.Kill().Wait()
-	})
-
-	It("starts and doesn't terminate immediately", func() {
-		Consistently(session).ShouldNot(gexec.Exit())
-	})
-
-	tryConnect := func() (net.Conn, error) {
-		programOutput := session.Out.Contents()
-		addr := strings.TrimSpace(string(programOutput))
-
-		conn, err := net.Dial("tcp", addr)
-		if err != nil {
-			return nil, err
-		}
-		return conn, err
-	}
-
-	It("prints its listening address to stdout", func() {
-		Eventually(session.Out).Should(gbytes.Say("\n"))
-		conn, err := tryConnect()
-		Expect(err).NotTo(HaveOccurred())
-		conn.Close()
-	})
-
-	It("will echo data back to us", func() {
-		Eventually(session.Out).Should(gbytes.Say("\n"))
-		conn, err := tryConnect()
-		Expect(err).NotTo(HaveOccurred())
-		defer conn.Close()
-
-		fmt.Fprintf(conn, "hello\n")
-		Expect(ioutil.ReadAll(conn)).To(Equal([]byte("hello")))
-	})
-})

pkg/testutils/netns_linux.go

@@ -53,7 +53,7 @@ func NewNS() (ns.NetNS, error) {
 	nsRunDir := getNsRunDir()
 
 	b := make([]byte, 16)
-	_, err := rand.Reader.Read(b)
+	_, err := rand.Read(b)
 	if err != nil {
 		return nil, fmt.Errorf("failed to generate random netns name: %v", err)
 	}

pkg/testutils/ping.go

@@ -17,13 +17,24 @@ package testutils
 import (
 	"bytes"
 	"fmt"
+	"net"
 	"os/exec"
 	"strconv"
 	"syscall"
 )
 
 // Ping shells out to the `ping` command. Returns nil if successful.
-func Ping(saddr, daddr string, isV6 bool, timeoutSec int) error {
+func Ping(saddr, daddr string, timeoutSec int) error {
+	ip := net.ParseIP(saddr)
+	if ip == nil {
+		return fmt.Errorf("failed to parse IP %q", saddr)
+	}
+
+	bin := "ping6"
+	if ip.To4() != nil {
+		bin = "ping"
+	}
+
 	args := []string{
 		"-c", "1",
 		"-W", strconv.Itoa(timeoutSec),
@@ -31,11 +42,6 @@ func Ping(saddr, daddr string, isV6 bool, timeoutSec int) error {
 		daddr,
 	}
 
-	bin := "ping"
-	if isV6 {
-		bin = "ping6"
-	}
-
 	cmd := exec.Command(bin, args...)
 	var stderr bytes.Buffer
 	cmd.Stderr = &stderr

pkg/testutils/testing.go

@@ -0,0 +1,54 @@
+// Copyright 2016 CNI authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package testutils
+
+import (
+	"github.com/containernetworking/cni/pkg/version"
+)
+
+// AllSpecVersions contains all CNI spec version numbers
+var AllSpecVersions = [...]string{"0.1.0", "0.2.0", "0.3.0", "0.3.1", "0.4.0", "1.0.0"}
+
+// SpecVersionHasIPVersion returns true if the given CNI specification version
+// includes the "version" field in the IP address elements
+func SpecVersionHasIPVersion(ver string) bool {
+	for _, i := range []string{"0.3.0", "0.3.1", "0.4.0"} {
+		if ver == i {
+			return true
+		}
+	}
+	return false
+}
+
+// SpecVersionHasCHECK returns true if the given CNI specification version
+// supports the CHECK command
+func SpecVersionHasCHECK(ver string) bool {
+	ok, _ := version.GreaterThanOrEqualTo(ver, "0.4.0")
+	return ok
+}
+
+// SpecVersionHasChaining returns true if the given CNI specification version
+// supports plugin chaining
+func SpecVersionHasChaining(ver string) bool {
+	ok, _ := version.GreaterThanOrEqualTo(ver, "0.3.0")
+	return ok
+}
+
+// SpecVersionHasMultipleIPs returns true if the given CNI specification version
+// supports more than one IP address of each family
+func SpecVersionHasMultipleIPs(ver string) bool {
+	ok, _ := version.GreaterThanOrEqualTo(ver, "0.3.0")
+	return ok
+}

pkg/utils/conntrack.go

@@ -0,0 +1,73 @@
+// Copyright 2020 CNI authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package utils
+
+import (
+	"fmt"
+	"net"
+
+	"github.com/vishvananda/netlink"
+	"golang.org/x/sys/unix"
+)
+
+// Assigned Internet Protocol Numbers
+// https://www.iana.org/assignments/protocol-numbers/protocol-numbers.xhtml
+const (
+	PROTOCOL_TCP  = 6
+	PROTOCOL_UDP  = 17
+	PROTOCOL_SCTP = 132
+)
+
+// getNetlinkFamily returns the Netlink IP family constant
+func getNetlinkFamily(isIPv6 bool) netlink.InetFamily {
+	if isIPv6 {
+		return unix.AF_INET6
+	}
+	return unix.AF_INET
+}
+
+// DeleteConntrackEntriesForDstIP delete the conntrack entries for the connections
+// specified by the given destination IP and protocol
+func DeleteConntrackEntriesForDstIP(dstIP string, protocol uint8) error {
+	ip := net.ParseIP(dstIP)
+	if ip == nil {
+		return fmt.Errorf("error deleting connection tracking state, bad IP %s", ip)
+	}
+	family := getNetlinkFamily(ip.To4() == nil)
+
+	filter := &netlink.ConntrackFilter{}
+	filter.AddIP(netlink.ConntrackOrigDstIP, ip)
+	filter.AddProtocol(protocol)
+
+	_, err := netlink.ConntrackDeleteFilter(netlink.ConntrackTable, family, filter)
+	if err != nil {
+		return fmt.Errorf("error deleting connection tracking state for protocol: %d IP: %s, error: %v", protocol, ip, err)
+	}
+	return nil
+}
+
+// DeleteConntrackEntriesForDstPort delete the conntrack entries for the connections specified
+// by the given destination port, protocol and IP family
+func DeleteConntrackEntriesForDstPort(port uint16, protocol uint8, family netlink.InetFamily) error {
+	filter := &netlink.ConntrackFilter{}
+	filter.AddProtocol(protocol)
+	filter.AddPort(netlink.ConntrackOrigDstPort, port)
+
+	_, err := netlink.ConntrackDeleteFilter(netlink.ConntrackTable, family, filter)
+	if err != nil {
+		return fmt.Errorf("error deleting connection tracking state for protocol: %d Port: %d, error: %v", protocol, port, err)
+	}
+	return nil
+}

pkg/utils/hwaddr/hwaddr.go

@@ -1,63 +0,0 @@
-// Copyright 2016 CNI authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package hwaddr
-
-import (
-	"fmt"
-	"net"
-)
-
-const (
-	ipRelevantByteLen      = 4
-	PrivateMACPrefixString = "0a:58"
-)
-
-var (
-	// private mac prefix safe to use
-	PrivateMACPrefix = []byte{0x0a, 0x58}
-)
-
-type SupportIp4OnlyErr struct{ msg string }
-
-func (e SupportIp4OnlyErr) Error() string { return e.msg }
-
-type MacParseErr struct{ msg string }
-
-func (e MacParseErr) Error() string { return e.msg }
-
-type InvalidPrefixLengthErr struct{ msg string }
-
-func (e InvalidPrefixLengthErr) Error() string { return e.msg }
-
-// GenerateHardwareAddr4 generates 48 bit virtual mac addresses based on the IP4 input.
-func GenerateHardwareAddr4(ip net.IP, prefix []byte) (net.HardwareAddr, error) {
-	switch {
-
-	case ip.To4() == nil:
-		return nil, SupportIp4OnlyErr{msg: "GenerateHardwareAddr4 only supports valid IPv4 address as input"}
-
-	case len(prefix) != len(PrivateMACPrefix):
-		return nil, InvalidPrefixLengthErr{msg: fmt.Sprintf(
-			"Prefix has length %d instead  of %d", len(prefix), len(PrivateMACPrefix)),
-		}
-	}
-
-	ipByteLen := len(ip)
-	return (net.HardwareAddr)(
-		append(
-			prefix,
-			ip[ipByteLen-ipRelevantByteLen:ipByteLen]...),
-	), nil
-}

pkg/utils/hwaddr/hwaddr_test.go

@@ -1,74 +0,0 @@
-// Copyright 2016 CNI authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package hwaddr_test
-
-import (
-	"net"
-
-	"github.com/containernetworking/plugins/pkg/utils/hwaddr"
-
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-)
-
-var _ = Describe("Hwaddr", func() {
-	Context("Generate Hardware Address", func() {
-		It("generate hardware address based on ipv4 address", func() {
-			testCases := []struct {
-				ip          net.IP
-				expectedMAC net.HardwareAddr
-			}{
-				{
-					ip:          net.ParseIP("10.0.0.2"),
-					expectedMAC: (net.HardwareAddr)(append(hwaddr.PrivateMACPrefix, 0x0a, 0x00, 0x00, 0x02)),
-				},
-				{
-					ip:          net.ParseIP("10.250.0.244"),
-					expectedMAC: (net.HardwareAddr)(append(hwaddr.PrivateMACPrefix, 0x0a, 0xfa, 0x00, 0xf4)),
-				},
-				{
-					ip:          net.ParseIP("172.17.0.2"),
-					expectedMAC: (net.HardwareAddr)(append(hwaddr.PrivateMACPrefix, 0xac, 0x11, 0x00, 0x02)),
-				},
-				{
-					ip:          net.IPv4(byte(172), byte(17), byte(0), byte(2)),
-					expectedMAC: (net.HardwareAddr)(append(hwaddr.PrivateMACPrefix, 0xac, 0x11, 0x00, 0x02)),
-				},
-			}
-
-			for _, tc := range testCases {
-				mac, err := hwaddr.GenerateHardwareAddr4(tc.ip, hwaddr.PrivateMACPrefix)
-				Expect(err).NotTo(HaveOccurred())
-				Expect(mac).To(Equal(tc.expectedMAC))
-			}
-		})
-
-		It("return error if input is not ipv4 address", func() {
-			testCases := []net.IP{
-				net.ParseIP(""),
-				net.ParseIP("2001:db8:0:1:1:1:1:1"),
-			}
-			for _, tc := range testCases {
-				_, err := hwaddr.GenerateHardwareAddr4(tc, hwaddr.PrivateMACPrefix)
-				Expect(err).To(BeAssignableToTypeOf(hwaddr.SupportIp4OnlyErr{}))
-			}
-		})
-
-		It("return error if prefix is invalid", func() {
-			_, err := hwaddr.GenerateHardwareAddr4(net.ParseIP("10.0.0.2"), []byte{0x58})
-			Expect(err).To(BeAssignableToTypeOf(hwaddr.InvalidPrefixLengthErr{}))
-		})
-	})
-})

pkg/utils/iptables.go

@@ -119,3 +119,21 @@ func ClearChain(ipt *iptables.IPTables, table, chain string) error {
 		return err
 	}
 }
+
+// InsertUnique will add a rule to a chain if it does not already exist.
+// By default the rule is appended, unless prepend is true.
+func InsertUnique(ipt *iptables.IPTables, table, chain string, prepend bool, rule []string) error {
+	exists, err := ipt.Exists(table, chain, rule...)
+	if err != nil {
+		return err
+	}
+	if exists {
+		return nil
+	}
+
+	if prepend {
+		return ipt.Insert(table, chain, 1, rule...)
+	} else {
+		return ipt.Append(table, chain, rule...)
+	}
+}

pkg/utils/sysctl/sysctl_linux.go

@@ -36,7 +36,6 @@ func Sysctl(name string, params ...string) (string, error) {
 
 func getSysctl(name string) (string, error) {
 	fullName := filepath.Join("/proc/sys", toNormalName(name))
-	fullName = filepath.Clean(fullName)
 	data, err := ioutil.ReadFile(fullName)
 	if err != nil {
 		return "", err
@@ -47,7 +46,6 @@ func getSysctl(name string) (string, error) {
 
 func setSysctl(name, value string) (string, error) {
 	fullName := filepath.Join("/proc/sys", toNormalName(name))
-	fullName = filepath.Clean(fullName)
 	if err := ioutil.WriteFile(fullName, []byte(value), 0644); err != nil {
 		return "", err
 	}

plugins/ipam/dhcp/README.md

@@ -1,39 +1,4 @@
-# dhcp plugin
 
-## Overview
+This document has moved to the [containernetworking/cni.dev](https://github.com/containernetworking/cni.dev) repo.
 
-With dhcp plugin the containers can get an IP allocated by a DHCP server already running on your network.
-This can be especially useful with plugin types such as [macvlan](../../main/macvlan/README.md).
-Because a DHCP lease must be periodically renewed for the duration of container lifetime, a separate daemon is required to be running.
-The same plugin binary can also be run in the daemon mode.
-
-## Operation
-To use the dhcp IPAM plugin, first launch the dhcp daemon:
-
-```
-# Make sure the unix socket has been removed
-$ rm -f /run/cni/dhcp.sock
-$ ./dhcp daemon
-```
-
-If given `-pidfile <path>` arguments after 'daemon', the dhcp plugin will write
-its PID to the given file.
-If given `-hostprefix <prefix>` arguments after 'daemon', the dhcp plugin will use this prefix for DHCP socket as `<prefix>/run/cni/dhcp.sock`. You can use this prefix for references to the host filesystem, e.g. to access netns and the unix socket.
-
-Alternatively, you can use systemd socket activation protocol.
-Be sure that the .socket file uses /run/cni/dhcp.sock as the socket path.
-
-With the daemon running, containers using the dhcp plugin can be launched.
-
-## Example configuration
-
-```
-{
-	"ipam": {
-		"type": "dhcp",
-	}
-}
-
-## Network configuration reference
-
-* `type` (string, required): "dhcp"
+You can find it online here: https://cni.dev/plugins/current/ipam/dhcp/

plugins/ipam/dhcp/client.go

@@ -1,3 +1,17 @@
+// Copyright 2021 CNI authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package main
 
 import (

plugins/ipam/dhcp/daemon.go

@@ -26,44 +26,61 @@ import (
 	"path/filepath"
 	"runtime"
 	"sync"
+	"time"
 
 	"github.com/containernetworking/cni/pkg/skel"
-	"github.com/containernetworking/cni/pkg/types"
-	"github.com/containernetworking/cni/pkg/types/current"
-	"github.com/coreos/go-systemd/activation"
+	current "github.com/containernetworking/cni/pkg/types/100"
+	"github.com/coreos/go-systemd/v22/activation"
 )
 
-const listenFdsStart = 3
-
 var errNoMoreTries = errors.New("no more tries")
 
 type DHCP struct {
 	mux             sync.Mutex
 	leases          map[string]*DHCPLease
 	hostNetnsPrefix string
+	clientTimeout   time.Duration
+	clientResendMax time.Duration
+	broadcast       bool
 }
 
-func newDHCP() *DHCP {
+func newDHCP(clientTimeout, clientResendMax time.Duration) *DHCP {
 	return &DHCP{
-		leases: make(map[string]*DHCPLease),
+		leases:          make(map[string]*DHCPLease),
+		clientTimeout:   clientTimeout,
+		clientResendMax: clientResendMax,
 	}
 }
 
+// TODO: current client ID is too long. At least the container ID should not be used directly.
+// A seperate issue is necessary to ensure no breaking change is affecting other users.
 func generateClientID(containerID string, netName string, ifName string) string {
-	return containerID + "/" + netName + "/" + ifName
+	clientID := containerID + "/" + netName + "/" + ifName
+	// defined in RFC 2132, length size can not be larger than 1 octet. So we truncate 254 to make everyone happy.
+	if len(clientID) > 254 {
+		clientID = clientID[0:254]
+	}
+	return clientID
 }
 
 // Allocate acquires an IP from a DHCP server for a specified container.
 // The acquired lease will be maintained until Release() is called.
 func (d *DHCP) Allocate(args *skel.CmdArgs, result *current.Result) error {
-	conf := types.NetConf{}
+	conf := NetConf{}
 	if err := json.Unmarshal(args.StdinData, &conf); err != nil {
 		return fmt.Errorf("error parsing netconf: %v", err)
 	}
 
+	optsRequesting, optsProviding, err := prepareOptions(args.Args, conf.IPAM.ProvideOptions, conf.IPAM.RequestOptions)
+	if err != nil {
+		return err
+	}
+
 	clientID := generateClientID(args.ContainerID, conf.Name, args.IfName)
 	hostNetns := d.hostNetnsPrefix + args.Netns
-	l, err := AcquireLease(clientID, hostNetns, args.IfName)
+	l, err := AcquireLease(clientID, hostNetns, args.IfName,
+		optsRequesting, optsProviding,
+		d.clientTimeout, d.clientResendMax, d.broadcast)
 	if err != nil {
 		return err
 	}
@@ -77,7 +94,6 @@ func (d *DHCP) Allocate(args *skel.CmdArgs, result *current.Result) error {
 	d.setLease(clientID, l)
 
 	result.IPs = []*current.IPConfig{{
-		Version: "4",
 		Address: *ipn,
 		Gateway: l.Gateway(),
 	}}
@@ -89,7 +105,7 @@ func (d *DHCP) Allocate(args *skel.CmdArgs, result *current.Result) error {
 // Release stops maintenance of the lease acquired in Allocate()
 // and sends a release msg to the DHCP server.
 func (d *DHCP) Release(args *skel.CmdArgs, reply *struct{}) error {
-	conf := types.NetConf{}
+	conf := NetConf{}
 	if err := json.Unmarshal(args.StdinData, &conf); err != nil {
 		return fmt.Errorf("error parsing netconf: %v", err)
 	}
@@ -156,7 +172,10 @@ func getListener(socketPath string) (net.Listener, error) {
 	}
 }
 
-func runDaemon(pidfilePath string, hostPrefix string, socketPath string) error {
+func runDaemon(
+	pidfilePath, hostPrefix, socketPath string,
+	dhcpClientTimeout time.Duration, resendMax time.Duration, broadcast bool,
+) error {
 	// since other goroutines (on separate threads) will change namespaces,
 	// ensure the RPC server does not get scheduled onto those
 	runtime.LockOSThread()
@@ -176,8 +195,9 @@ func runDaemon(pidfilePath string, hostPrefix string, socketPath string) error {
 		return fmt.Errorf("Error getting listener: %v", err)
 	}
 
-	dhcp := newDHCP()
+	dhcp := newDHCP(dhcpClientTimeout, resendMax)
 	dhcp.hostNetnsPrefix = hostPrefix
+	dhcp.broadcast = broadcast
 	rpc.Register(dhcp)
 	rpc.HandleHTTP()
 	http.Serve(l, nil)

plugins/ipam/dhcp/dhcp2_test.go

@@ -23,7 +23,7 @@ import (
 	"time"
 
 	"github.com/containernetworking/cni/pkg/skel"
-	"github.com/containernetworking/cni/pkg/types/current"
+	current "github.com/containernetworking/cni/pkg/types/100"
 	"github.com/containernetworking/plugins/pkg/ns"
 	"github.com/containernetworking/plugins/pkg/testutils"
 

plugins/ipam/dhcp/dhcp_test.go

@@ -15,7 +15,9 @@
 package main
 
 import (
+	"bytes"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"net"
 	"os"
@@ -25,7 +27,7 @@ import (
 	"time"
 
 	"github.com/containernetworking/cni/pkg/skel"
-	"github.com/containernetworking/cni/pkg/types/current"
+	"github.com/containernetworking/cni/pkg/types/100"
 	"github.com/containernetworking/plugins/pkg/ns"
 	"github.com/containernetworking/plugins/pkg/testutils"
 
@@ -208,6 +210,11 @@ var _ = Describe("DHCP Operations", func() {
 		dhcpPluginPath, err := exec.LookPath("dhcp")
 		Expect(err).NotTo(HaveOccurred())
 		clientCmd = exec.Command(dhcpPluginPath, "daemon", "-socketpath", socketPath)
+
+		// copy dhcp client's stdout/stderr to test stdout
+		clientCmd.Stdout = os.Stdout
+		clientCmd.Stderr = os.Stderr
+
 		err = clientCmd.Start()
 		Expect(err).NotTo(HaveOccurred())
 		Expect(clientCmd.Process).NotTo(BeNil())
@@ -226,118 +233,127 @@ var _ = Describe("DHCP Operations", func() {
 		clientCmd.Wait()
 
 		Expect(originalNS.Close()).To(Succeed())
+		Expect(testutils.UnmountNS(originalNS)).To(Succeed())
 		Expect(targetNS.Close()).To(Succeed())
-		defer os.RemoveAll(tmpDir)
+		Expect(testutils.UnmountNS(targetNS)).To(Succeed())
+
+		Expect(os.RemoveAll(tmpDir)).To(Succeed())
 	})
 
-	It("configures and deconfigures a link with ADD/DEL", func() {
-		conf := fmt.Sprintf(`{
-    "cniVersion": "0.3.1",
-    "name": "mynet",
-    "type": "ipvlan",
-    "ipam": {
-        "type": "dhcp",
-	"daemonSocketPath": "%s"
-    }
-}`, socketPath)
+	for _, ver := range testutils.AllSpecVersions {
+		// Redefine ver inside for scope so real value is picked up by each dynamically defined It()
+		// See Gingkgo's "Patterns for dynamically generating tests" documentation.
+		ver := ver
 
-		args := &skel.CmdArgs{
-			ContainerID: "dummy",
-			Netns:       targetNS.Path(),
-			IfName:      contVethName,
-			StdinData:   []byte(conf),
-		}
+		It(fmt.Sprintf("[%s] configures and deconfigures a link with ADD/DEL", ver), func() {
+			conf := fmt.Sprintf(`{
+			    "cniVersion": "%s",
+			    "name": "mynet",
+			    "type": "ipvlan",
+			    "ipam": {
+				"type": "dhcp",
+				"daemonSocketPath": "%s"
+			    }
+			}`, ver, socketPath)
 
-		var addResult *current.Result
-		err := originalNS.Do(func(ns.NetNS) error {
-			defer GinkgoRecover()
+			args := &skel.CmdArgs{
+				ContainerID: "dummy",
+				Netns:       targetNS.Path(),
+				IfName:      contVethName,
+				StdinData:   []byte(conf),
+			}
 
-			r, _, err := testutils.CmdAddWithArgs(args, func() error {
-				return cmdAdd(args)
-			})
-			Expect(err).NotTo(HaveOccurred())
-
-			addResult, err = current.GetResult(r)
-			Expect(err).NotTo(HaveOccurred())
-			Expect(len(addResult.IPs)).To(Equal(1))
-			Expect(addResult.IPs[0].Address.String()).To(Equal("192.168.1.5/24"))
-			return nil
-		})
-		Expect(err).NotTo(HaveOccurred())
-
-		err = originalNS.Do(func(ns.NetNS) error {
-			return testutils.CmdDelWithArgs(args, func() error {
-				return cmdDel(args)
-			})
-		})
-		Expect(err).NotTo(HaveOccurred())
-	})
-
-	It("correctly handles multiple DELs for the same container", func() {
-		conf := fmt.Sprintf(`{
-    "cniVersion": "0.3.1",
-    "name": "mynet",
-    "type": "ipvlan",
-    "ipam": {
-        "type": "dhcp",
-	"daemonSocketPath": "%s"
-    }
-}`, socketPath)
-
-		args := &skel.CmdArgs{
-			ContainerID: "dummy",
-			Netns:       targetNS.Path(),
-			IfName:      contVethName,
-			StdinData:   []byte(conf),
-		}
-
-		var addResult *current.Result
-		err := originalNS.Do(func(ns.NetNS) error {
-			defer GinkgoRecover()
-
-			r, _, err := testutils.CmdAddWithArgs(args, func() error {
-				return cmdAdd(args)
-			})
-			Expect(err).NotTo(HaveOccurred())
-
-			addResult, err = current.GetResult(r)
-			Expect(err).NotTo(HaveOccurred())
-			Expect(len(addResult.IPs)).To(Equal(1))
-			Expect(addResult.IPs[0].Address.String()).To(Equal("192.168.1.5/24"))
-			return nil
-		})
-		Expect(err).NotTo(HaveOccurred())
-
-		wg := sync.WaitGroup{}
-		wg.Add(3)
-		started := sync.WaitGroup{}
-		started.Add(3)
-		for i := 0; i < 3; i++ {
-			go func() {
+			var addResult *types100.Result
+			err := originalNS.Do(func(ns.NetNS) error {
 				defer GinkgoRecover()
 
-				// Wait until all goroutines are running
-				started.Done()
-				started.Wait()
-
-				err = originalNS.Do(func(ns.NetNS) error {
-					return testutils.CmdDelWithArgs(args, func() error {
-						return cmdDel(args)
-					})
+				r, _, err := testutils.CmdAddWithArgs(args, func() error {
+					return cmdAdd(args)
 				})
 				Expect(err).NotTo(HaveOccurred())
-				wg.Done()
-			}()
-		}
-		wg.Wait()
 
-		err = originalNS.Do(func(ns.NetNS) error {
-			return testutils.CmdDelWithArgs(args, func() error {
-				return cmdDel(args)
+				addResult, err = types100.GetResult(r)
+				Expect(err).NotTo(HaveOccurred())
+				Expect(len(addResult.IPs)).To(Equal(1))
+				Expect(addResult.IPs[0].Address.String()).To(Equal("192.168.1.5/24"))
+				return nil
 			})
+			Expect(err).NotTo(HaveOccurred())
+
+			err = originalNS.Do(func(ns.NetNS) error {
+				return testutils.CmdDelWithArgs(args, func() error {
+					return cmdDel(args)
+				})
+			})
+			Expect(err).NotTo(HaveOccurred())
 		})
-		Expect(err).NotTo(HaveOccurred())
-	})
+
+		It(fmt.Sprintf("[%s] correctly handles multiple DELs for the same container", ver), func() {
+			conf := fmt.Sprintf(`{
+			    "cniVersion": "%s",
+			    "name": "mynet",
+			    "type": "ipvlan",
+			    "ipam": {
+				"type": "dhcp",
+				"daemonSocketPath": "%s"
+			    }
+			}`, ver, socketPath)
+
+			args := &skel.CmdArgs{
+				ContainerID: "dummy",
+				Netns:       targetNS.Path(),
+				IfName:      contVethName,
+				StdinData:   []byte(conf),
+			}
+
+			var addResult *types100.Result
+			err := originalNS.Do(func(ns.NetNS) error {
+				defer GinkgoRecover()
+
+				r, _, err := testutils.CmdAddWithArgs(args, func() error {
+					return cmdAdd(args)
+				})
+				Expect(err).NotTo(HaveOccurred())
+
+				addResult, err = types100.GetResult(r)
+				Expect(err).NotTo(HaveOccurred())
+				Expect(len(addResult.IPs)).To(Equal(1))
+				Expect(addResult.IPs[0].Address.String()).To(Equal("192.168.1.5/24"))
+				return nil
+			})
+			Expect(err).NotTo(HaveOccurred())
+
+			wg := sync.WaitGroup{}
+			wg.Add(3)
+			started := sync.WaitGroup{}
+			started.Add(3)
+			for i := 0; i < 3; i++ {
+				go func() {
+					defer GinkgoRecover()
+
+					// Wait until all goroutines are running
+					started.Done()
+					started.Wait()
+
+					err = originalNS.Do(func(ns.NetNS) error {
+						return testutils.CmdDelWithArgs(args, func() error {
+							return cmdDel(args)
+						})
+					})
+					Expect(err).NotTo(HaveOccurred())
+					wg.Done()
+				}()
+			}
+			wg.Wait()
+
+			err = originalNS.Do(func(ns.NetNS) error {
+				return testutils.CmdDelWithArgs(args, func() error {
+					return cmdDel(args)
+				})
+			})
+			Expect(err).NotTo(HaveOccurred())
+		})
+	}
 })
 
 const (
@@ -508,7 +524,19 @@ var _ = Describe("DHCP Lease Unavailable Operations", func() {
 		// Start the DHCP client daemon
 		dhcpPluginPath, err := exec.LookPath("dhcp")
 		Expect(err).NotTo(HaveOccurred())
-		clientCmd = exec.Command(dhcpPluginPath, "daemon", "-socketpath", socketPath)
+		// Use very short timeouts for lease-unavailable operations because
+		// the same test is run many times, and the delays will exceed the
+		// `go test` timeout with default delays. Since our DHCP server
+		// and client daemon are local processes anyway, we can depend on
+		// them to respond very quickly.
+		clientCmd = exec.Command(dhcpPluginPath, "daemon", "-socketpath", socketPath, "-timeout", "2s", "-resendmax", "8s")
+
+		// copy dhcp client's stdout/stderr to test stdout
+		var b bytes.Buffer
+		mw := io.MultiWriter(os.Stdout, &b)
+		clientCmd.Stdout = mw
+		clientCmd.Stderr = mw
+
 		err = clientCmd.Start()
 		Expect(err).NotTo(HaveOccurred())
 		Expect(clientCmd.Process).NotTo(BeNil())
@@ -527,92 +555,101 @@ var _ = Describe("DHCP Lease Unavailable Operations", func() {
 		clientCmd.Wait()
 
 		Expect(originalNS.Close()).To(Succeed())
+		Expect(testutils.UnmountNS(originalNS)).To(Succeed())
 		Expect(targetNS.Close()).To(Succeed())
-		defer os.RemoveAll(tmpDir)
+		Expect(testutils.UnmountNS(targetNS)).To(Succeed())
+
+		Expect(os.RemoveAll(tmpDir)).To(Succeed())
 	})
 
-	It("Configures multiple links with multiple ADD with second lease unavailable", func() {
-		conf := fmt.Sprintf(`{
-	    "cniVersion": "0.3.1",
-	    "name": "mynet",
-	    "type": "bridge",
-	    "bridge": "%s",
-	    "ipam": {
-	        "type": "dhcp",
-		"daemonSocketPath": "%s"
-	    }
-	}`, hostBridgeName, socketPath)
+	for _, ver := range testutils.AllSpecVersions {
+		// Redefine ver inside for scope so real value is picked up by each dynamically defined It()
+		// See Gingkgo's "Patterns for dynamically generating tests" documentation.
+		ver := ver
 
-		args := &skel.CmdArgs{
-			ContainerID: "dummy",
-			Netns:       targetNS.Path(),
-			IfName:      contVethName0,
-			StdinData:   []byte(conf),
-		}
+		It(fmt.Sprintf("[%s] configures multiple links with multiple ADD with second lease unavailable", ver), func() {
+			conf := fmt.Sprintf(`{
+			    "cniVersion": "%s",
+			    "name": "mynet",
+			    "type": "bridge",
+			    "bridge": "%s",
+			    "ipam": {
+				"type": "dhcp",
+				"daemonSocketPath": "%s"
+			    }
+			}`, ver, hostBridgeName, socketPath)
 
-		var addResult *current.Result
-		err := originalNS.Do(func(ns.NetNS) error {
-			defer GinkgoRecover()
+			args := &skel.CmdArgs{
+				ContainerID: "dummy",
+				Netns:       targetNS.Path(),
+				IfName:      contVethName0,
+				StdinData:   []byte(conf),
+			}
 
-			r, _, err := testutils.CmdAddWithArgs(args, func() error {
-				return cmdAdd(args)
+			var addResult *types100.Result
+			err := originalNS.Do(func(ns.NetNS) error {
+				defer GinkgoRecover()
+
+				r, _, err := testutils.CmdAddWithArgs(args, func() error {
+					return cmdAdd(args)
+				})
+				Expect(err).NotTo(HaveOccurred())
+
+				addResult, err = types100.GetResult(r)
+				Expect(err).NotTo(HaveOccurred())
+				Expect(len(addResult.IPs)).To(Equal(1))
+				Expect(addResult.IPs[0].Address.String()).To(Equal("192.168.1.5/24"))
+				return nil
 			})
 			Expect(err).NotTo(HaveOccurred())
 
-			addResult, err = current.GetResult(r)
+			args = &skel.CmdArgs{
+				ContainerID: "dummy",
+				Netns:       targetNS.Path(),
+				IfName:      contVethName1,
+				StdinData:   []byte(conf),
+			}
+
+			err = originalNS.Do(func(ns.NetNS) error {
+				defer GinkgoRecover()
+
+				_, _, err := testutils.CmdAddWithArgs(args, func() error {
+					return cmdAdd(args)
+				})
+				Expect(err).To(HaveOccurred())
+				println(err.Error())
+				Expect(err.Error()).To(Equal("error calling DHCP.Allocate: no more tries"))
+				return nil
+			})
 			Expect(err).NotTo(HaveOccurred())
-			Expect(len(addResult.IPs)).To(Equal(1))
-			Expect(addResult.IPs[0].Address.String()).To(Equal("192.168.1.5/24"))
-			return nil
-		})
-		Expect(err).NotTo(HaveOccurred())
 
-		args = &skel.CmdArgs{
-			ContainerID: "dummy",
-			Netns:       targetNS.Path(),
-			IfName:      contVethName1,
-			StdinData:   []byte(conf),
-		}
+			args = &skel.CmdArgs{
+				ContainerID: "dummy",
+				Netns:       targetNS.Path(),
+				IfName:      contVethName1,
+				StdinData:   []byte(conf),
+			}
 
-		err = originalNS.Do(func(ns.NetNS) error {
-			defer GinkgoRecover()
-
-			_, _, err := testutils.CmdAddWithArgs(args, func() error {
-				return cmdAdd(args)
+			err = originalNS.Do(func(ns.NetNS) error {
+				return testutils.CmdDelWithArgs(args, func() error {
+					return cmdDel(args)
+				})
 			})
-			Expect(err).To(HaveOccurred())
-			println(err.Error())
-			Expect(err.Error()).To(Equal("error calling DHCP.Allocate: no more tries"))
-			return nil
-		})
-		Expect(err).NotTo(HaveOccurred())
+			Expect(err).NotTo(HaveOccurred())
 
-		args = &skel.CmdArgs{
-			ContainerID: "dummy",
-			Netns:       targetNS.Path(),
-			IfName:      contVethName1,
-			StdinData:   []byte(conf),
-		}
+			args = &skel.CmdArgs{
+				ContainerID: "dummy",
+				Netns:       targetNS.Path(),
+				IfName:      contVethName0,
+				StdinData:   []byte(conf),
+			}
 
-		err = originalNS.Do(func(ns.NetNS) error {
-			return testutils.CmdDelWithArgs(args, func() error {
-				return cmdDel(args)
+			err = originalNS.Do(func(ns.NetNS) error {
+				return testutils.CmdDelWithArgs(args, func() error {
+					return cmdDel(args)
+				})
 			})
+			Expect(err).NotTo(HaveOccurred())
 		})
-		Expect(err).NotTo(HaveOccurred())
-
-		args = &skel.CmdArgs{
-			ContainerID: "dummy",
-			Netns:       targetNS.Path(),
-			IfName:      contVethName0,
-			StdinData:   []byte(conf),
-		}
-
-		err = originalNS.Do(func(ns.NetNS) error {
-			return testutils.CmdDelWithArgs(args, func() error {
-				return cmdDel(args)
-			})
-		})
-		Expect(err).NotTo(HaveOccurred())
-	})
+	}
 })

plugins/ipam/dhcp/lease.go

@@ -19,6 +19,7 @@ import (
 	"log"
 	"math/rand"
 	"net"
+	"strings"
 	"sync"
 	"sync/atomic"
 	"time"
@@ -36,6 +37,11 @@ import (
 const resendDelay0 = 4 * time.Second
 const resendDelayMax = 62 * time.Second
 
+// To speed up the retry for first few failures, we retry without
+// backoff for a few times
+const resendFastDelay = 2 * time.Second
+const resendFastMax = 4
+
 const (
 	leaseStateBound = iota
 	leaseStateRenewing
@@ -56,19 +62,99 @@ type DHCPLease struct {
 	renewalTime   time.Time
 	rebindingTime time.Time
 	expireTime    time.Time
+	timeout       time.Duration
+	resendMax     time.Duration
+	broadcast     bool
 	stopping      uint32
 	stop          chan struct{}
 	wg            sync.WaitGroup
+	// list of requesting and providing options and if they are necessary / their value
+	optsRequesting map[dhcp4.OptionCode]bool
+	optsProviding  map[dhcp4.OptionCode][]byte
+}
+
+var requestOptionsDefault = map[dhcp4.OptionCode]bool{
+	dhcp4.OptionRouter:     true,
+	dhcp4.OptionSubnetMask: true,
+}
+
+func prepareOptions(cniArgs string, ProvideOptions []ProvideOption, RequestOptions []RequestOption) (
+	optsRequesting map[dhcp4.OptionCode]bool, optsProviding map[dhcp4.OptionCode][]byte, err error) {
+
+	// parse CNI args
+	cniArgsParsed := map[string]string{}
+	for _, argPair := range strings.Split(cniArgs, ";") {
+		args := strings.SplitN(argPair, "=", 2)
+		if len(args) > 1 {
+			cniArgsParsed[args[0]] = args[1]
+		}
+	}
+
+	// parse providing options map
+	var optParsed dhcp4.OptionCode
+	optsProviding = make(map[dhcp4.OptionCode][]byte)
+	for _, opt := range ProvideOptions {
+		optParsed, err = parseOptionName(string(opt.Option))
+		if err != nil {
+			err = fmt.Errorf("Can not parse option %q: %w", opt.Option, err)
+			return
+		}
+		if len(opt.Value) > 0 {
+			if len(opt.Value) > 255 {
+				err = fmt.Errorf("value too long for option %q: %q", opt.Option, opt.Value)
+				return
+			}
+			optsProviding[optParsed] = []byte(opt.Value)
+		}
+		if value, ok := cniArgsParsed[opt.ValueFromCNIArg]; ok {
+			if len(value) > 255 {
+				err = fmt.Errorf("value too long for option %q from CNI_ARGS %q: %q", opt.Option, opt.ValueFromCNIArg, opt.Value)
+				return
+			}
+			optsProviding[optParsed] = []byte(value)
+		}
+	}
+
+	// parse necessary options map
+	optsRequesting = make(map[dhcp4.OptionCode]bool)
+	skipRequireDefault := false
+	for _, opt := range RequestOptions {
+		if opt.SkipDefault {
+			skipRequireDefault = true
+		}
+		optParsed, err = parseOptionName(string(opt.Option))
+		if err != nil {
+			err = fmt.Errorf("Can not parse option %q: %w", opt.Option, err)
+			return
+		}
+		optsRequesting[optParsed] = true
+	}
+	for k, v := range requestOptionsDefault {
+		// only set if not skipping default and this value does not exists
+		if _, ok := optsRequesting[k]; !ok && !skipRequireDefault {
+			optsRequesting[k] = v
+		}
+	}
+	return
 }
 
 // AcquireLease gets an DHCP lease and then maintains it in the background
 // by periodically renewing it. The acquired lease can be released by
 // calling DHCPLease.Stop()
-func AcquireLease(clientID, netns, ifName string) (*DHCPLease, error) {
+func AcquireLease(
+	clientID, netns, ifName string,
+	optsRequesting map[dhcp4.OptionCode]bool, optsProviding map[dhcp4.OptionCode][]byte,
+	timeout, resendMax time.Duration, broadcast bool,
+) (*DHCPLease, error) {
 	errCh := make(chan error, 1)
 	l := &DHCPLease{
-		clientID: clientID,
-		stop:     make(chan struct{}),
+		clientID:       clientID,
+		stop:           make(chan struct{}),
+		timeout:        timeout,
+		resendMax:      resendMax,
+		broadcast:      broadcast,
+		optsRequesting: optsRequesting,
+		optsProviding:  optsProviding,
 	}
 
 	log.Printf("%v: acquiring lease", clientID)
@@ -115,7 +201,7 @@ func (l *DHCPLease) Stop() {
 }
 
 func (l *DHCPLease) acquire() error {
-	c, err := newDHCPClient(l.link, l.clientID)
+	c, err := newDHCPClient(l.link, l.clientID, l.timeout, l.broadcast)
 	if err != nil {
 		return err
 	}
@@ -130,9 +216,19 @@ func (l *DHCPLease) acquire() error {
 
 	opts := make(dhcp4.Options)
 	opts[dhcp4.OptionClientIdentifier] = []byte(l.clientID)
-	opts[dhcp4.OptionParameterRequestList] = []byte{byte(dhcp4.OptionRouter), byte(dhcp4.OptionSubnetMask)}
+	opts[dhcp4.OptionParameterRequestList] = []byte{}
+	for k := range l.optsRequesting {
+		opts[dhcp4.OptionParameterRequestList] = append(opts[dhcp4.OptionParameterRequestList], byte(k))
+	}
+	for k, v := range l.optsProviding {
+		opts[k] = v
+	}
+	// client identifier's first byte is "type"
+	newClientID := []byte{0}
+	newClientID = append(newClientID, opts[dhcp4.OptionClientIdentifier]...)
+	opts[dhcp4.OptionClientIdentifier] = newClientID
 
-	pkt, err := backoffRetry(func() (*dhcp4.Packet, error) {
+	pkt, err := backoffRetry(l.resendMax, func() (*dhcp4.Packet, error) {
 		ok, ack, err := DhcpRequest(c, opts)
 		switch {
 		case err != nil:
@@ -242,7 +338,7 @@ func (l *DHCPLease) downIface() {
 }
 
 func (l *DHCPLease) renew() error {
-	c, err := newDHCPClient(l.link, l.clientID)
+	c, err := newDHCPClient(l.link, l.clientID, l.timeout, l.broadcast)
 	if err != nil {
 		return err
 	}
@@ -251,7 +347,7 @@ func (l *DHCPLease) renew() error {
 	opts := make(dhcp4.Options)
 	opts[dhcp4.OptionClientIdentifier] = []byte(l.clientID)
 
-	pkt, err := backoffRetry(func() (*dhcp4.Packet, error) {
+	pkt, err := backoffRetry(l.resendMax, func() (*dhcp4.Packet, error) {
 		ok, ack, err := DhcpRenew(c, *l.ack, opts)
 		switch {
 		case err != nil:
@@ -273,7 +369,7 @@ func (l *DHCPLease) renew() error {
 func (l *DHCPLease) release() error {
 	log.Printf("%v: releasing lease", l.clientID)
 
-	c, err := newDHCPClient(l.link, l.clientID)
+	c, err := newDHCPClient(l.link, l.clientID, l.timeout, l.broadcast)
 	if err != nil {
 		return err
 	}
@@ -333,10 +429,10 @@ func jitter(span time.Duration) time.Duration {
 	return time.Duration(float64(span) * (2.0*rand.Float64() - 1.0))
 }
 
-func backoffRetry(f func() (*dhcp4.Packet, error)) (*dhcp4.Packet, error) {
+func backoffRetry(resendMax time.Duration, f func() (*dhcp4.Packet, error)) (*dhcp4.Packet, error) {
 	var baseDelay time.Duration = resendDelay0
 	var sleepTime time.Duration
-
+	var fastRetryLimit = resendFastMax
 	for {
 		pkt, err := f()
 		if err == nil {
@@ -345,15 +441,21 @@ func backoffRetry(f func() (*dhcp4.Packet, error)) (*dhcp4.Packet, error) {
 
 		log.Print(err)
 
-		sleepTime = baseDelay + jitter(time.Second)
+		if fastRetryLimit == 0 {
+			sleepTime = baseDelay + jitter(time.Second)
+		} else {
+			sleepTime = resendFastDelay + jitter(time.Second)
+			fastRetryLimit--
+		}
 
 		log.Printf("retrying in %f seconds", sleepTime.Seconds())
 
 		time.Sleep(sleepTime)
 
-		if baseDelay < resendDelayMax {
+		// only adjust delay time if we are in normal backoff stage
+		if baseDelay < resendMax && fastRetryLimit == 0 {
 			baseDelay *= 2
-		} else {
+		} else if fastRetryLimit == 0 { // only break if we are at normal delay
 			break
 		}
 	}
@@ -361,7 +463,11 @@ func backoffRetry(f func() (*dhcp4.Packet, error)) (*dhcp4.Packet, error) {
 	return nil, errNoMoreTries
 }
 
-func newDHCPClient(link netlink.Link, clientID string) (*dhcp4client.Client, error) {
+func newDHCPClient(
+	link netlink.Link, clientID string,
+	timeout time.Duration,
+	broadcast bool,
+) (*dhcp4client.Client, error) {
 	pktsock, err := dhcp4client.NewPacketSock(link.Attrs().Index)
 	if err != nil {
 		return nil, err
@@ -369,8 +475,8 @@ func newDHCPClient(link netlink.Link, clientID string) (*dhcp4client.Client, err
 
 	return dhcp4client.New(
 		dhcp4client.HardwareAddr(link.Attrs().HardwareAddr),
-		dhcp4client.Timeout(5*time.Second),
-		dhcp4client.Broadcast(false),
+		dhcp4client.Timeout(timeout),
+		dhcp4client.Broadcast(broadcast),
 		dhcp4client.Connection(pktsock),
 	)
 }

plugins/ipam/dhcp/main.go

@@ -22,33 +22,77 @@ import (
 	"net/rpc"
 	"os"
 	"path/filepath"
+	"time"
 
 	"github.com/containernetworking/cni/pkg/skel"
 	"github.com/containernetworking/cni/pkg/types"
-	"github.com/containernetworking/cni/pkg/types/current"
+	current "github.com/containernetworking/cni/pkg/types/100"
 	"github.com/containernetworking/cni/pkg/version"
 	bv "github.com/containernetworking/plugins/pkg/utils/buildversion"
 )
 
 const defaultSocketPath = "/run/cni/dhcp.sock"
 
+// The top-level network config - IPAM plugins are passed the full configuration
+// of the calling plugin, not just the IPAM section.
+type NetConf struct {
+	types.NetConf
+	IPAM *IPAMConfig `json:"ipam"`
+}
+
+type IPAMConfig struct {
+	types.IPAM
+	DaemonSocketPath string `json:"daemonSocketPath"`
+	// When requesting IP from DHCP server, carry these options for management purpose.
+	// Some fields have default values, and can be override by setting a new option with the same name at here.
+	ProvideOptions []ProvideOption `json:"provide"`
+	// When requesting IP from DHCP server, claiming these options are necessary. Options are necessary unless `optional`
+	// is set to `false`.
+	// To override default requesting fields, set `skipDefault` to `false`.
+	// If an field is not optional, but the server failed to provide it, error will be raised.
+	RequestOptions []RequestOption `json:"request"`
+}
+
+// DHCPOption represents a DHCP option. It can be a number, or a string defined in manual dhcp-options(5).
+// Note that not all DHCP options are supported at all time. Error will be raised if unsupported options are used.
+type DHCPOption string
+
+type ProvideOption struct {
+	Option DHCPOption `json:"option"`
+
+	Value           string `json:"value"`
+	ValueFromCNIArg string `json:"fromArg"`
+}
+
+type RequestOption struct {
+	SkipDefault bool `json:"skipDefault"`
+
+	Option DHCPOption `json:"option"`
+}
+
 func main() {
 	if len(os.Args) > 1 && os.Args[1] == "daemon" {
 		var pidfilePath string
 		var hostPrefix string
 		var socketPath string
+		var broadcast bool
+		var timeout time.Duration
+		var resendMax time.Duration
 		daemonFlags := flag.NewFlagSet("daemon", flag.ExitOnError)
 		daemonFlags.StringVar(&pidfilePath, "pidfile", "", "optional path to write daemon PID to")
 		daemonFlags.StringVar(&hostPrefix, "hostprefix", "", "optional prefix to host root")
 		daemonFlags.StringVar(&socketPath, "socketpath", "", "optional dhcp server socketpath")
+		daemonFlags.BoolVar(&broadcast, "broadcast", false, "broadcast DHCP leases")
+		daemonFlags.DurationVar(&timeout, "timeout", 10*time.Second, "optional dhcp client timeout duration")
+		daemonFlags.DurationVar(&resendMax, "resendmax", resendDelayMax, "optional dhcp client resend max duration")
 		daemonFlags.Parse(os.Args[2:])
 
 		if socketPath == "" {
 			socketPath = defaultSocketPath
 		}
 
-		if err := runDaemon(pidfilePath, hostPrefix, socketPath); err != nil {
-			log.Printf(err.Error())
+		if err := runDaemon(pidfilePath, hostPrefix, socketPath, timeout, resendMax, broadcast); err != nil {
+			log.Print(err.Error())
 			os.Exit(1)
 		}
 	} else {
@@ -64,7 +108,7 @@ func cmdAdd(args *skel.CmdArgs) error {
 		return err
 	}
 
-	result := &current.Result{}
+	result := &current.Result{CNIVersion: current.ImplementedSpecVersion}
 	if err := rpcCall("DHCP.Allocate", args, result); err != nil {
 		return err
 	}
@@ -81,8 +125,6 @@ func cmdDel(args *skel.CmdArgs) error {
 }
 
 func cmdCheck(args *skel.CmdArgs) error {
-	// TODO: implement
-	//return fmt.Errorf("not implemented")
 	// Plugin must return result in same version as specified in netconf
 	versionDecoder := &version.ConfigDecoder{}
 	//confVersion, err := versionDecoder.Decode(args.StdinData)
@@ -91,7 +133,7 @@ func cmdCheck(args *skel.CmdArgs) error {
 		return err
 	}
 
-	result := &current.Result{}
+	result := &current.Result{CNIVersion: current.ImplementedSpecVersion}
 	if err := rpcCall("DHCP.Allocate", args, result); err != nil {
 		return err
 	}
@@ -99,16 +141,8 @@ func cmdCheck(args *skel.CmdArgs) error {
 	return nil
 }
 
-type SocketPathConf struct {
-	DaemonSocketPath string `json:"daemonSocketPath,omitempty"`
-}
-
-type TempNetConf struct {
-	IPAM SocketPathConf `json:"ipam,omitempty"`
-}
-
 func getSocketPath(stdinData []byte) (string, error) {
-	conf := TempNetConf{}
+	conf := NetConf{}
 	if err := json.Unmarshal(stdinData, &conf); err != nil {
 		return "", fmt.Errorf("error parsing socket path conf: %v", err)
 	}

plugins/ipam/dhcp/options.go

@@ -18,12 +18,33 @@ import (
 	"encoding/binary"
 	"fmt"
 	"net"
+	"strconv"
 	"time"
 
 	"github.com/containernetworking/cni/pkg/types"
 	"github.com/d2g/dhcp4"
 )
 
+var optionNameToID = map[string]dhcp4.OptionCode{
+	"dhcp-client-identifier":  dhcp4.OptionClientIdentifier,
+	"subnet-mask":             dhcp4.OptionSubnetMask,
+	"routers":                 dhcp4.OptionRouter,
+	"host-name":               dhcp4.OptionHostName,
+	"user-class":              dhcp4.OptionUserClass,
+	"vendor-class-identifier": dhcp4.OptionVendorClassIdentifier,
+}
+
+func parseOptionName(option string) (dhcp4.OptionCode, error) {
+	if val, ok := optionNameToID[option]; ok {
+		return val, nil
+	}
+	i, err := strconv.ParseUint(option, 10, 8)
+	if err != nil {
+		return 0, fmt.Errorf("Can not parse option: %w", err)
+	}
+	return dhcp4.OptionCode(i), nil
+}
+
 func parseRouter(opts dhcp4.Options) net.IP {
 	if opts, ok := opts[dhcp4.OptionRouter]; ok {
 		if len(opts) == 4 {

plugins/ipam/dhcp/options_test.go

@@ -16,6 +16,7 @@ package main
 
 import (
 	"net"
+	"reflect"
 	"testing"
 
 	"github.com/containernetworking/cni/pkg/types"
@@ -73,3 +74,34 @@ func TestParseCIDRRoutes(t *testing.T) {
 
 	validateRoutes(t, routes)
 }
+
+func TestParseOptionName(t *testing.T) {
+	tests := []struct {
+		name    string
+		option  string
+		want    dhcp4.OptionCode
+		wantErr bool
+	}{
+		{
+			"hostname", "host-name", dhcp4.OptionHostName, false,
+		},
+		{
+			"hostname in number", "12", dhcp4.OptionHostName, false,
+		},
+		{
+			"random string", "doNotparseMe", 0, true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := parseOptionName(tt.option)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("parseOptionName() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("parseOptionName() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

plugins/ipam/host-local/README.md

@@ -1,142 +1,4 @@
-# host-local IP address management plugin
 
-host-local IPAM allocates IPv4 and IPv6 addresses out of a specified address range. Optionally,
-it can include a DNS configuration from a `resolv.conf` file on the host.
+This document has moved to the [containernetworking/cni.dev](https://github.com/containernetworking/cni.dev) repo.
 
-## Overview
-
-host-local IPAM plugin allocates ip addresses out of a set of address ranges.
-It stores the state locally on the host filesystem, therefore ensuring uniqueness of IP addresses on a single host.
-
-The allocator can allocate multiple ranges, and supports sets of multiple (disjoint) 
-subnets. The allocation strategy is loosely round-robin within each range set.
-
-## Example configurations
-
-Note that the key `ranges` is a list of range sets. That is to say, the length 
-of the top-level array is the number of addresses returned. The second-level 
-array is a set of subnets to use as a pool of possible addresses.
-
-This example configuration returns 2 IP addresses.
-
-```json
-{
-	"ipam": {
-		"type": "host-local",
-		"ranges": [
-			[
-				{
-					"subnet": "10.10.0.0/16",
-					"rangeStart": "10.10.1.20",
-					"rangeEnd": "10.10.3.50",
-					"gateway": "10.10.0.254"
-				},
-				{
-					"subnet": "172.16.5.0/24"
-				}
-			],
-			[
-				{
-					"subnet": "3ffe:ffff:0:01ff::/64",
-					"rangeStart": "3ffe:ffff:0:01ff::0010",
-					"rangeEnd": "3ffe:ffff:0:01ff::0020"
-				}
-			]
-		],
-		"routes": [
-			{ "dst": "0.0.0.0/0" },
-			{ "dst": "192.168.0.0/16", "gw": "10.10.5.1" },
-			{ "dst": "3ffe:ffff:0:01ff::1/64" }
-		],
-		"dataDir": "/run/my-orchestrator/container-ipam-state"
-	}
-}
-```
-
-Previous versions of the `host-local` allocator did not support the `ranges`
-property, and instead expected a single range on the top level. This is
-deprecated but still supported.
-```json
-{
-  "ipam": {
-		"type": "host-local",
-		"subnet": "3ffe:ffff:0:01ff::/64",
-		"rangeStart": "3ffe:ffff:0:01ff::0010",
-		"rangeEnd": "3ffe:ffff:0:01ff::0020",
-		"routes": [
-			{ "dst": "3ffe:ffff:0:01ff::1/64" }
-		],
-		"resolvConf": "/etc/resolv.conf"
-	}
-}
-```
-
-We can test it out on the command-line:
-
-```bash
-$ echo '{ "cniVersion": "0.3.1", "name": "examplenet", "ipam": { "type": "host-local", "ranges": [ [{"subnet": "203.0.113.0/24"}], [{"subnet": "2001:db8:1::/64"}]], "dataDir": "/tmp/cni-example"  } }' | CNI_COMMAND=ADD CNI_CONTAINERID=example CNI_NETNS=/dev/null CNI_IFNAME=dummy0 CNI_PATH=. ./host-local
-
-```
-
-```json
-{
-    "ips": [
-        {
-            "version": "4",
-            "address": "203.0.113.2/24",
-            "gateway": "203.0.113.1"
-        },
-        {
-            "version": "6",
-            "address": "2001:db8:1::2/64",
-            "gateway": "2001:db8:1::1"
-        }
-    ],
-    "dns": {}
-}
-```
-
-## Network configuration reference
-
-* `type` (string, required): "host-local".
-* `routes` (string, optional): list of routes to add to the container namespace. Each route is a dictionary with "dst" and optional "gw" fields. If "gw" is omitted, value of "gateway" will be used.
-* `resolvConf` (string, optional): Path to a `resolv.conf` on the host to parse and return as the DNS configuration
-* `dataDir` (string, optional): Path to a directory to use for maintaining state, e.g. which IPs have been allocated to which containers
-* `ranges`, (array, required, nonempty) an array of arrays of range objects:
-	* `subnet` (string, required): CIDR block to allocate out of.
-	* `rangeStart` (string, optional): IP inside of "subnet" from which to start allocating addresses. Defaults to ".2" IP inside of the "subnet" block.
-	* `rangeEnd` (string, optional): IP inside of "subnet" with which to end allocating addresses. Defaults to ".254" IP inside of the "subnet" block for ipv4, ".255" for IPv6
-	* `gateway` (string, optional): IP inside of "subnet" to designate as the gateway. Defaults to ".1" IP inside of the "subnet" block.
-
-Older versions of the `host-local` plugin did not support the `ranges` array. Instead,
-all the properties in  the `range` object were top-level. This is still supported but deprecated.
-
-## Supported arguments
-The following [CNI_ARGS](https://github.com/containernetworking/cni/blob/master/SPEC.md#parameters) are supported:
-
-* `ip`: request a specific IP address from a subnet.
-
-The following [args conventions](https://github.com/containernetworking/cni/blob/master/CONVENTIONS.md) are supported:
-
-* `ips` (array of strings): A list of custom IPs to attempt to allocate
-
-The following [Capability Args](https://github.com/containernetworking/cni/blob/master/CONVENTIONS.md) are supported:
-
-* `ipRanges`: The exact same as the `ranges` array - a list of address pools
-
-### Custom IP allocation
-For every requested custom IP, the `host-local` allocator will request that IP
-if it falls within one of the `range` objects. Thus it is possible to specify
-multiple custom IPs and multiple ranges.
-
-If any requested IPs cannot be reserved, either because they are already in use
-or are not part of a specified range, the plugin will return an error.
-
-
-## Files
-
-Allocated IP addresses are stored as files in `/var/lib/cni/networks/$NETWORK_NAME`.
-The path can be customized with the `dataDir` option listed above. Environments
-where IPs are released automatically on reboot (e.g. running containers are not
-restored) may wish to specify `/var/run/cni` or another tmpfs mounted directory
-instead.
+You can find it online here: https://cni.dev/plugins/current/ipam/host-local/

plugins/ipam/host-local/backend/allocator/allocator.go

@@ -21,7 +21,8 @@ import (
 	"os"
 	"strconv"
 
-	"github.com/containernetworking/cni/pkg/types/current"
+	current "github.com/containernetworking/cni/pkg/types/100"
+
 	"github.com/containernetworking/plugins/pkg/ip"
 	"github.com/containernetworking/plugins/plugins/ipam/host-local/backend"
 )
@@ -108,13 +109,8 @@ func (a *IPAllocator) Get(id string, ifname string, requestedIP net.IP) (*curren
 	if reservedIP == nil {
 		return nil, fmt.Errorf("no IP addresses available in range set: %s", a.rangeset.String())
 	}
-	version := "4"
-	if reservedIP.IP.To4() == nil {
-		version = "6"
-	}
 
 	return &current.IPConfig{
-		Version: version,
 		Address: *reservedIP,
 		Gateway: gw,
 	}, nil
@@ -137,9 +133,8 @@ type RangeIter struct {
 	// Our current position
 	cur net.IP
 
-	// The IP and range index where we started iterating; if we hit this again, we're done.
-	startIP    net.IP
-	startRange int
+	// The IP where we started iterating; if we hit this again, we're done.
+	startIP net.IP
 }
 
 // GetIter encapsulates the strategy for this allocator.
@@ -169,7 +164,6 @@ func (a *IPAllocator) GetIter() (*RangeIter, error) {
 		for i, r := range *a.rangeset {
 			if r.Contains(lastReservedIP) {
 				iter.rangeIdx = i
-				iter.startRange = i
 
 				// We advance the cursor on every Next(), so the first call
 				// to next() will return lastReservedIP + 1
@@ -179,7 +173,6 @@ func (a *IPAllocator) GetIter() (*RangeIter, error) {
 		}
 	} else {
 		iter.rangeIdx = 0
-		iter.startRange = 0
 		iter.startIP = (*a.rangeset)[0].RangeStart
 	}
 	return &iter, nil
@@ -215,7 +208,7 @@ func (i *RangeIter) Next() (*net.IPNet, net.IP) {
 
 	if i.startIP == nil {
 		i.startIP = i.cur
-	} else if i.rangeIdx == i.startRange && i.cur.Equal(i.startIP) {
+	} else if i.cur.Equal(i.startIP) {
 		// IF we've looped back to where we started, give up
 		return nil, nil
 	}

plugins/ipam/host-local/backend/allocator/allocator_test.go

@@ -19,7 +19,7 @@ import (
 	"net"
 
 	"github.com/containernetworking/cni/pkg/types"
-	"github.com/containernetworking/cni/pkg/types/current"
+	current "github.com/containernetworking/cni/pkg/types/100"
 	fakestore "github.com/containernetworking/plugins/plugins/ipam/host-local/backend/testing"
 
 	. "github.com/onsi/ginkgo"
@@ -49,6 +49,23 @@ func mkalloc() IPAllocator {
 	return alloc
 }
 
+func newAllocatorWithMultiRanges() IPAllocator {
+	p := RangeSet{
+		Range{RangeStart: net.IP{192, 168, 1, 0}, RangeEnd: net.IP{192, 168, 1, 3}, Subnet: mustSubnet("192.168.1.1/30")},
+		Range{RangeStart: net.IP{192, 168, 2, 0}, RangeEnd: net.IP{192, 168, 2, 3}, Subnet: mustSubnet("192.168.2.1/30")},
+	}
+	_ = p.Canonicalize()
+	store := fakestore.NewFakeStore(map[string]string{}, map[string]net.IP{})
+
+	alloc := IPAllocator{
+		rangeset: &p,
+		store:    store,
+		rangeID:  "rangeid",
+	}
+
+	return alloc
+}
+
 func (t AllocatorTestCase) run(idx int) (*current.IPConfig, error) {
 	fmt.Fprintln(GinkgoWriter, "Index:", idx)
 	p := RangeSet{}
@@ -320,6 +337,39 @@ var _ = Describe("host-local ip allocator", func() {
 			}
 		})
 	})
+
+	Context("when lastReservedIP is at the end of one of multi ranges", func() {
+		It("should use the first IP of next range as startIP after Next", func() {
+			a := newAllocatorWithMultiRanges()
+
+			// reserve the last IP of the first range
+			reserved, err := a.store.Reserve("ID", "eth0", net.IP{192, 168, 1, 3}, a.rangeID)
+			Expect(reserved).To(BeTrue())
+			Expect(err).NotTo(HaveOccurred())
+
+			// get range iterator and do the first Next
+			r, err := a.GetIter()
+			Expect(err).NotTo(HaveOccurred())
+			ip := r.nextip()
+			Expect(ip).NotTo(BeNil())
+
+			Expect(r.startIP).To(Equal(net.IP{192, 168, 2, 0}))
+		})
+	})
+
+	Context("when no lastReservedIP", func() {
+		It("should use the first IP of the first range as startIP after Next", func() {
+			a := newAllocatorWithMultiRanges()
+
+			// get range iterator and do the first Next
+			r, err := a.GetIter()
+			Expect(err).NotTo(HaveOccurred())
+			ip := r.nextip()
+			Expect(ip).NotTo(BeNil())
+
+			Expect(r.startIP).To(Equal(net.IP{192, 168, 1, 0}))
+		})
+	})
 })
 
 // nextip is a convenience function used for testing

plugins/ipam/host-local/backend/allocator/config.go

@@ -20,7 +20,9 @@ import (
 	"net"
 
 	"github.com/containernetworking/cni/pkg/types"
-	"github.com/containernetworking/cni/pkg/types/020"
+	"github.com/containernetworking/cni/pkg/version"
+
+	"github.com/containernetworking/plugins/pkg/ip"
 )
 
 // The top-level network config - IPAM plugins are passed the full configuration
@@ -29,8 +31,10 @@ type Net struct {
 	Name          string      `json:"name"`
 	CNIVersion    string      `json:"cniVersion"`
 	IPAM          *IPAMConfig `json:"ipam"`
-	RuntimeConfig struct {    // The capability arg
+	RuntimeConfig struct {
+		// The capability arg
 		IPRanges []RangeSet `json:"ipRanges,omitempty"`
+		IPs      []*ip.IP   `json:"ips,omitempty"`
 	} `json:"runtimeConfig,omitempty"`
 	Args *struct {
 		A *IPAMArgs `json:"cni"`
@@ -48,16 +52,16 @@ type IPAMConfig struct {
 	DataDir    string         `json:"dataDir"`
 	ResolvConf string         `json:"resolvConf"`
 	Ranges     []RangeSet     `json:"ranges"`
-	IPArgs     []net.IP       `json:"-"` // Requested IPs from CNI_ARGS and args
+	IPArgs     []net.IP       `json:"-"` // Requested IPs from CNI_ARGS, args and capabilities
 }
 
 type IPAMEnvArgs struct {
 	types.CommonArgs
-	IP net.IP `json:"ip,omitempty"`
+	IP ip.IP `json:"ip,omitempty"`
 }
 
 type IPAMArgs struct {
-	IPs []net.IP `json:"ips"`
+	IPs []*ip.IP `json:"ips"`
 }
 
 type RangeSet []Range
@@ -80,7 +84,7 @@ func LoadIPAMConfig(bytes []byte, envArgs string) (*IPAMConfig, string, error) {
 		return nil, "", fmt.Errorf("IPAM config missing 'ipam' key")
 	}
 
-	// Parse custom IP from both env args *and* the top-level args config
+	// parse custom IP from env args
 	if envArgs != "" {
 		e := IPAMEnvArgs{}
 		err := types.LoadArgs(envArgs, &e)
@@ -88,13 +92,23 @@ func LoadIPAMConfig(bytes []byte, envArgs string) (*IPAMConfig, string, error) {
 			return nil, "", err
 		}
 
-		if e.IP != nil {
-			n.IPAM.IPArgs = []net.IP{e.IP}
+		if e.IP.ToIP() != nil {
+			n.IPAM.IPArgs = []net.IP{e.IP.ToIP()}
 		}
 	}
 
+	// parse custom IPs from CNI args in network config
 	if n.Args != nil && n.Args.A != nil && len(n.Args.A.IPs) != 0 {
-		n.IPAM.IPArgs = append(n.IPAM.IPArgs, n.Args.A.IPs...)
+		for _, i := range n.Args.A.IPs {
+			n.IPAM.IPArgs = append(n.IPAM.IPArgs, i.ToIP())
+		}
+	}
+
+	// parse custom IPs from runtime configuration
+	if len(n.RuntimeConfig.IPs) > 0 {
+		for _, i := range n.RuntimeConfig.IPs {
+			n.IPAM.IPArgs = append(n.IPAM.IPArgs, i.ToIP())
+		}
 	}
 
 	for idx := range n.IPAM.IPArgs {
@@ -136,10 +150,8 @@ func LoadIPAMConfig(bytes []byte, envArgs string) (*IPAMConfig, string, error) {
 
 	// CNI spec 0.2.0 and below supported only one v4 and v6 address
 	if numV4 > 1 || numV6 > 1 {
-		for _, v := range types020.SupportedVersions {
-			if n.CNIVersion == v {
-				return nil, "", fmt.Errorf("CNI version %v does not support more than 1 address per family", n.CNIVersion)
-			}
+		if ok, _ := version.GreaterThanOrEqualTo(n.CNIVersion, "0.3.0"); !ok {
+			return nil, "", fmt.Errorf("CNI version %v does not support more than 1 address per family", n.CNIVersion)
 		}
 	}
 

plugins/ipam/host-local/backend/allocator/config_test.go

@@ -205,8 +205,9 @@ var _ = Describe("IPAM config", func() {
 		}))
 	})
 
-	It("Should parse CNI_ARGS env", func() {
-		input := `{
+	Context("Should parse CNI_ARGS env", func() {
+		It("without prefix", func() {
+			input := `{
 			"cniVersion": "0.3.1",
 			"name": "mynet",
 			"type": "ipvlan",
@@ -224,16 +225,43 @@ var _ = Describe("IPAM config", func() {
 			}
 		}`
 
-		envArgs := "IP=10.1.2.10"
+			envArgs := "IP=10.1.2.10"
 
-		conf, _, err := LoadIPAMConfig([]byte(input), envArgs)
-		Expect(err).NotTo(HaveOccurred())
-		Expect(conf.IPArgs).To(Equal([]net.IP{{10, 1, 2, 10}}))
+			conf, _, err := LoadIPAMConfig([]byte(input), envArgs)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(conf.IPArgs).To(Equal([]net.IP{{10, 1, 2, 10}}))
+		})
 
+		It("with prefix", func() {
+			input := `{
+			"cniVersion": "0.3.1",
+			"name": "mynet",
+			"type": "ipvlan",
+			"master": "foo0",
+			"ipam": {
+				"type": "host-local",
+				"ranges": [[
+					{
+						"subnet": "10.1.2.0/24",
+						"rangeStart": "10.1.2.9",
+						"rangeEnd": "10.1.2.20",
+						"gateway": "10.1.2.30"
+					}
+				]]
+			}
+		}`
+
+			envArgs := "IP=10.1.2.11/24"
+
+			conf, _, err := LoadIPAMConfig([]byte(input), envArgs)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(conf.IPArgs).To(Equal([]net.IP{{10, 1, 2, 11}}))
+		})
 	})
 
-	It("Should parse config args", func() {
-		input := `{
+	Context("Should parse config args", func() {
+		It("without prefix", func() {
+			input := `{
 			"cniVersion": "0.3.1",
 			"name": "mynet",
 			"type": "ipvlan",
@@ -265,16 +293,62 @@ var _ = Describe("IPAM config", func() {
 			}
 		}`
 
-		envArgs := "IP=10.1.2.10"
+			envArgs := "IP=10.1.2.10"
 
-		conf, _, err := LoadIPAMConfig([]byte(input), envArgs)
-		Expect(err).NotTo(HaveOccurred())
-		Expect(conf.IPArgs).To(Equal([]net.IP{
-			{10, 1, 2, 10},
-			{10, 1, 2, 11},
-			{11, 11, 11, 11},
-			net.ParseIP("2001:db8:1::11"),
-		}))
+			conf, _, err := LoadIPAMConfig([]byte(input), envArgs)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(conf.IPArgs).To(Equal([]net.IP{
+				{10, 1, 2, 10},
+				{10, 1, 2, 11},
+				{11, 11, 11, 11},
+				net.ParseIP("2001:db8:1::11"),
+			}))
+		})
+
+		It("with prefix", func() {
+			input := `{
+			"cniVersion": "0.3.1",
+			"name": "mynet",
+			"type": "ipvlan",
+			"master": "foo0",
+			"args": {
+				"cni": {
+					"ips": [ "10.1.2.11/24", "11.11.11.11/24", "2001:db8:1::11/64"]
+				}
+			},
+			"ipam": {
+				"type": "host-local",
+				"ranges": [
+					[{
+						"subnet": "10.1.2.0/24",
+						"rangeStart": "10.1.2.9",
+						"rangeEnd": "10.1.2.20",
+						"gateway": "10.1.2.30"
+					}],
+					[{
+						"subnet": "11.1.2.0/24",
+						"rangeStart": "11.1.2.9",
+						"rangeEnd": "11.1.2.20",
+						"gateway": "11.1.2.30"
+					}],
+					[{
+						"subnet": "2001:db8:1::/64"
+					}]
+				]
+			}
+		}`
+
+			envArgs := "IP=10.1.2.10/24"
+
+			conf, _, err := LoadIPAMConfig([]byte(input), envArgs)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(conf.IPArgs).To(Equal([]net.IP{
+				{10, 1, 2, 10},
+				{10, 1, 2, 11},
+				{11, 11, 11, 11},
+				net.ParseIP("2001:db8:1::11"),
+			}))
+		})
 	})
 
 	It("Should detect overlap between rangesets", func() {
@@ -379,4 +453,29 @@ var _ = Describe("IPAM config", func() {
 		_, _, err := LoadIPAMConfig([]byte(input), "")
 		Expect(err).NotTo(HaveOccurred())
 	})
+
+	It("Should parse custom IPs from runtime configuration", func() {
+		input := `{
+			"cniVersion": "0.3.1",
+			"name": "mynet",
+			"type": "ipvlan",
+			"master": "foo0",
+			"runtimeConfig": {
+				"ips": ["192.168.0.1", "192.168.0.5/24", "2001:db8::1/64"]
+			},
+			"ipam": {
+				"type": "host-local",
+				"subnet": "10.1.2.0/24"
+			}
+		}`
+		conf, version, err := LoadIPAMConfig([]byte(input), "")
+		Expect(err).NotTo(HaveOccurred())
+		Expect(version).Should(Equal("0.3.1"))
+
+		Expect(conf.IPArgs).To(Equal([]net.IP{
+			net.IPv4(192, 168, 0, 1).To4(),
+			net.IPv4(192, 168, 0, 5).To4(),
+			net.ParseIP("2001:db8::1"),
+		}))
+	})
 })

plugins/ipam/host-local/main.go

@@ -15,7 +15,6 @@
 package main
 
 import (
-	"encoding/json"
 	"fmt"
 	"net"
 	"strings"
@@ -26,7 +25,7 @@ import (
 
 	"github.com/containernetworking/cni/pkg/skel"
 	"github.com/containernetworking/cni/pkg/types"
-	"github.com/containernetworking/cni/pkg/types/current"
+	current "github.com/containernetworking/cni/pkg/types/100"
 	"github.com/containernetworking/cni/pkg/version"
 )
 
@@ -34,14 +33,6 @@ func main() {
 	skel.PluginMain(cmdAdd, cmdCheck, cmdDel, version.All, bv.BuildString("host-local"))
 }
 
-func loadNetConf(bytes []byte) (*types.NetConf, string, error) {
-	n := &types.NetConf{}
-	if err := json.Unmarshal(bytes, n); err != nil {
-		return nil, "", fmt.Errorf("failed to load netconf: %v", err)
-	}
-	return n, n.CNIVersion, nil
-}
-
 func cmdCheck(args *skel.CmdArgs) error {
 
 	ipamConf, _, err := allocator.LoadIPAMConfig(args.StdinData, args.Args)
@@ -71,7 +62,7 @@ func cmdAdd(args *skel.CmdArgs) error {
 		return err
 	}
 
-	result := &current.Result{}
+	result := &current.Result{CNIVersion: current.ImplementedSpecVersion}
 
 	if ipamConf.ResolvConf != "" {
 		dns, err := parseResolvConf(ipamConf.ResolvConf)

plugins/ipam/static/README.md

@@ -1,68 +1,4 @@
-# static IP address management plugin
 
-## Overview
+This document has moved to the [containernetworking/cni.dev](https://github.com/containernetworking/cni.dev) repo.
 
-static IPAM is very simple IPAM plugin that assigns IPv4 and IPv6 addresses statically to container. This will be useful in debugging purpose and in case of assign same IP address in different vlan/vxlan to containers.
-
-
-## Example configuration
-
-```
-{
-	"ipam": {
-		"type": "static",
-		"addresses": [
-			{
-				"address": "10.10.0.1/24",
-				"gateway": "10.10.0.254"
-			},
-			{
-				"address": "3ffe:ffff:0:01ff::1/64",
-				"gateway": "3ffe:ffff:0::1"
-			}
-		],
-		"routes": [
-			{ "dst": "0.0.0.0/0" },
-			{ "dst": "192.168.0.0/16", "gw": "10.10.5.1" },
-			{ "dst": "3ffe:ffff:0:01ff::1/64" }
-		],
-		"dns": {
-			"nameservers" : ["8.8.8.8"],
-			"domain": "example.com",
-			"search": [ "example.com" ]
-		}
-	}
-}
-```
-
-## Network configuration reference
-
-* `type` (string, required): "static"
-* `addresses` (array, optional): an array of ip address objects:
-	* `address` (string, required): CIDR notation IP address.
-	* `gateway` (string, optional): IP inside of "subnet" to designate as the gateway.
-* `routes` (string, optional): list of routes add to the container namespace. Each route is a dictionary with "dst" and optional "gw" fields. If "gw" is omitted, value of "gateway" will be used.
-* `dns` (string, optional): the dictionary with "nameservers", "domain" and "search".
-
-## Supported arguments
-
-The following [CNI_ARGS](https://github.com/containernetworking/cni/blob/master/SPEC.md#parameters) are supported:
-
-* `IP`: request a specific CIDR notation IP addresses, comma separated
-* `GATEWAY`: request a specific gateway address
-
-    (example: CNI_ARGS="IP=10.10.0.1/24;GATEWAY=10.10.0.254")
-
-The plugin also support following [capability argument](https://github.com/containernetworking/cni/blob/master/CONVENTIONS.md).
-
-* `ips`: Pass IP addresses for CNI interface
-
-The following [args conventions](https://github.com/containernetworking/cni/blob/master/CONVENTIONS.md#args-in-network-config) are supported:
-
-* `ips` (array of strings): A list of custom IPs to attempt to allocate, with prefix (e.g. '10.10.0.1/24')
-
-Notice: If some of above are used at same time, only one will work according to the priorities below
-
-1. [capability argument](https://github.com/containernetworking/cni/blob/master/CONVENTIONS.md)
-1. [args conventions](https://github.com/containernetworking/cni/blob/master/CONVENTIONS.md#args-in-network-config)
-1. [CNI_ARGS](https://github.com/containernetworking/cni/blob/master/SPEC.md#parameters)
+You can find it online here: https://cni.dev/plugins/current/ipam/static/

plugins/ipam/static/main.go

@@ -22,8 +22,7 @@ import (
 
 	"github.com/containernetworking/cni/pkg/skel"
 	"github.com/containernetworking/cni/pkg/types"
-	types020 "github.com/containernetworking/cni/pkg/types/020"
-	"github.com/containernetworking/cni/pkg/types/current"
+	current "github.com/containernetworking/cni/pkg/types/100"
 	"github.com/containernetworking/cni/pkg/version"
 	bv "github.com/containernetworking/plugins/pkg/utils/buildversion"
 )
@@ -144,6 +143,9 @@ func LoadIPAMConfig(bytes []byte, envArgs string) (*IPAMConfig, string, error) {
 	if err := json.Unmarshal(bytes, &n); err != nil {
 		return nil, "", err
 	}
+	if n.IPAM == nil {
+		return nil, "", fmt.Errorf("IPAM config missing 'ipam' key")
+	}
 
 	// load IP from CNI_ARGS
 	if envArgs != "" {
@@ -159,7 +161,7 @@ func LoadIPAMConfig(bytes []byte, envArgs string) (*IPAMConfig, string, error) {
 
 				ip, subnet, err := net.ParseCIDR(ipstr)
 				if err != nil {
-					return nil, "", fmt.Errorf("invalid CIDR %s: %s", ipstr, err)
+					return nil, "", fmt.Errorf("the 'ip' field is expected to be in CIDR notation, got: '%s'", ipstr)
 				}
 
 				addr := Address{
@@ -190,8 +192,13 @@ func LoadIPAMConfig(bytes []byte, envArgs string) (*IPAMConfig, string, error) {
 	if n.Args != nil && n.Args.A != nil && len(n.Args.A.IPs) != 0 {
 		// args IP overwrites IP, so clear IPAM Config
 		n.IPAM.Addresses = make([]Address, 0, len(n.Args.A.IPs))
-		for _, addr := range n.Args.A.IPs {
-			n.IPAM.Addresses = append(n.IPAM.Addresses, Address{AddressStr: addr})
+		for _, addrStr := range n.Args.A.IPs {
+			ip, addr, err := net.ParseCIDR(addrStr)
+			if err != nil {
+				return nil, "", fmt.Errorf("an entry in the 'ips' field is NOT in CIDR notation, got: '%s'", addrStr)
+			}
+			addr.IP = ip
+			n.IPAM.Addresses = append(n.IPAM.Addresses, Address{AddressStr: addrStr, Address: *addr})
 		}
 	}
 
@@ -199,46 +206,46 @@ func LoadIPAMConfig(bytes []byte, envArgs string) (*IPAMConfig, string, error) {
 	if len(n.RuntimeConfig.IPs) != 0 {
 		// runtimeConfig IP overwrites IP, so clear IPAM Config
 		n.IPAM.Addresses = make([]Address, 0, len(n.RuntimeConfig.IPs))
-		for _, addr := range n.RuntimeConfig.IPs {
-			n.IPAM.Addresses = append(n.IPAM.Addresses, Address{AddressStr: addr})
+		for _, addrStr := range n.RuntimeConfig.IPs {
+			ip, addr, err := net.ParseCIDR(addrStr)
+			if err != nil {
+				return nil, "", fmt.Errorf("an entry in the 'ips' field is NOT in CIDR notation, got: '%s'", addrStr)
+			}
+			addr.IP = ip
+			n.IPAM.Addresses = append(n.IPAM.Addresses, Address{AddressStr: addrStr, Address: *addr})
 		}
 	}
 
-	if n.IPAM == nil {
-		return nil, "", fmt.Errorf("IPAM config missing 'ipam' key")
-	}
-
 	// Validate all ranges
 	numV4 := 0
 	numV6 := 0
 
 	for i := range n.IPAM.Addresses {
-		ip, addr, err := net.ParseCIDR(n.IPAM.Addresses[i].AddressStr)
-		if err != nil {
-			return nil, "", fmt.Errorf("invalid CIDR %s: %s", n.IPAM.Addresses[i].AddressStr, err)
+		if n.IPAM.Addresses[i].Address.IP == nil {
+			ip, addr, err := net.ParseCIDR(n.IPAM.Addresses[i].AddressStr)
+			if err != nil {
+				return nil, "", fmt.Errorf(
+					"the 'address' field is expected to be in CIDR notation, got: '%s'", n.IPAM.Addresses[i].AddressStr)
+			}
+			n.IPAM.Addresses[i].Address = *addr
+			n.IPAM.Addresses[i].Address.IP = ip
 		}
-		n.IPAM.Addresses[i].Address = *addr
-		n.IPAM.Addresses[i].Address.IP = ip
 
 		if err := canonicalizeIP(&n.IPAM.Addresses[i].Address.IP); err != nil {
 			return nil, "", fmt.Errorf("invalid address %d: %s", i, err)
 		}
 
 		if n.IPAM.Addresses[i].Address.IP.To4() != nil {
-			n.IPAM.Addresses[i].Version = "4"
 			numV4++
 		} else {
-			n.IPAM.Addresses[i].Version = "6"
 			numV6++
 		}
 	}
 
 	// CNI spec 0.2.0 and below supported only one v4 and v6 address
 	if numV4 > 1 || numV6 > 1 {
-		for _, v := range types020.SupportedVersions {
-			if n.CNIVersion == v {
-				return nil, "", fmt.Errorf("CNI version %v does not support more than 1 address per family", n.CNIVersion)
-			}
+		if ok, _ := version.GreaterThanOrEqualTo(n.CNIVersion, "0.3.0"); !ok {
+			return nil, "", fmt.Errorf("CNI version %v does not support more than 1 address per family", n.CNIVersion)
 		}
 	}
 
@@ -254,14 +261,16 @@ func cmdAdd(args *skel.CmdArgs) error {
 		return err
 	}
 
-	result := &current.Result{}
-	result.DNS = ipamConf.DNS
-	result.Routes = ipamConf.Routes
+	result := &current.Result{
+		CNIVersion: current.ImplementedSpecVersion,
+		DNS:        ipamConf.DNS,
+		Routes:     ipamConf.Routes,
+	}
 	for _, v := range ipamConf.Addresses {
 		result.IPs = append(result.IPs, &current.IPConfig{
-			Version: v.Version,
 			Address: v.Address,
-			Gateway: v.Gateway})
+			Gateway: v.Gateway,
+		})
 	}
 
 	return types.PrintResult(result, confVersion)

plugins/linux_only.txt

@@ -10,3 +10,4 @@ plugins/meta/portmap
 plugins/meta/tuning
 plugins/meta/bandwidth
 plugins/meta/firewall
+plugins/meta/vrf

plugins/main/bridge/README.md

@@ -1,60 +1,5 @@
-# bridge plugin
 
-## Overview
+This document has moved to the [containernetworking/cni.dev](https://github.com/containernetworking/cni.dev) repo.
 
-With bridge plugin, all containers (on the same host) are plugged into a bridge (virtual switch) that resides in the host network namespace.
-The containers receive one end of the veth pair with the other end connected to the bridge.
-An IP address is only assigned to one end of the veth pair -- one residing in the container.
-The bridge itself can also be assigned an IP address, turning it into a gateway for the containers.
-Alternatively, the bridge can function purely in L2 mode and would need to be bridged to the host network interface (if other than container-to-container communication on the same host is desired).
+You can find it online here: https://cni.dev/plugins/current/main/bridge/
 
-The network configuration specifies the name of the bridge to be used.
-If the bridge is missing, the plugin will create one on first use and, if gateway mode is used, assign it an IP that was returned by IPAM plugin via the gateway field.
-
-## Example configuration
-```
-{
-    "cniVersion": "0.3.1",
-    "name": "mynet",
-    "type": "bridge",
-    "bridge": "mynet0",
-    "isDefaultGateway": true,
-    "forceAddress": false,
-    "ipMasq": true,
-    "hairpinMode": true,
-    "ipam": {
-        "type": "host-local",
-        "subnet": "10.10.0.0/16"
-    }
-}
-```
-
-## Example L2-only configuration
-```
-{
-    "cniVersion": "0.3.1",
-    "name": "mynet",
-    "type": "bridge",
-    "bridge": "mynet0",
-    "ipam": {}
-}
-```
-
-## Network configuration reference
-
-* `name` (string, required): the name of the network.
-* `type` (string, required): "bridge".
-* `bridge` (string, optional): name of the bridge to use/create. Defaults to "cni0".
-* `isGateway` (boolean, optional): assign an IP address to the bridge. Defaults to false.
-* `isDefaultGateway` (boolean, optional): Sets isGateway to true and makes the assigned IP the default route. Defaults to false.
-* `forceAddress` (boolean, optional): Indicates if a new IP address should be set if the previous value has been changed. Defaults to false.
-* `ipMasq` (boolean, optional): set up IP Masquerade on the host for traffic originating from this network and destined outside of it. Defaults to false.
-* `mtu` (integer, optional): explicitly set MTU to the specified value. Defaults to the value chosen by the kernel.
-* `hairpinMode` (boolean, optional): set hairpin mode for interfaces on the bridge. Defaults to false.
-* `ipam` (dictionary, required): IPAM configuration to be used for this network. For L2-only network, create empty dictionary.
-* `promiscMode` (boolean, optional): set promiscuous mode on the bridge. Defaults to false.
-* `vlan` (int, optional): assign VLAN tag. Defaults to none.
-
-*Note:* The VLAN parameter configures the VLAN tag on the host end of the veth and also enables the vlan_filtering feature on the bridge interface.
-
-*Note:* To configure uplink for L2 network you need to allow the vlan on the uplink interface by using the following command ``` bridge vlan add vid VLAN_ID dev DEV```.

plugins/main/bridge/bridge.go

@@ -18,21 +18,21 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"io/ioutil"
 	"net"
+	"os"
 	"runtime"
 	"syscall"
 	"time"
 
-	"github.com/j-keck/arping"
 	"github.com/vishvananda/netlink"
 
 	"github.com/containernetworking/cni/pkg/skel"
 	"github.com/containernetworking/cni/pkg/types"
-	"github.com/containernetworking/cni/pkg/types/current"
+	current "github.com/containernetworking/cni/pkg/types/100"
 	"github.com/containernetworking/cni/pkg/version"
 	"github.com/containernetworking/plugins/pkg/ip"
 	"github.com/containernetworking/plugins/pkg/ipam"
+	"github.com/containernetworking/plugins/pkg/link"
 	"github.com/containernetworking/plugins/pkg/ns"
 	"github.com/containernetworking/plugins/pkg/utils"
 	bv "github.com/containernetworking/plugins/pkg/utils/buildversion"
@@ -55,6 +55,27 @@ type NetConf struct {
 	HairpinMode  bool   `json:"hairpinMode"`
 	PromiscMode  bool   `json:"promiscMode"`
 	Vlan         int    `json:"vlan"`
+	MacSpoofChk  bool   `json:"macspoofchk,omitempty"`
+	EnableDad    bool   `json:"enabledad,omitempty"`
+
+	Args struct {
+		Cni BridgeArgs `json:"cni,omitempty"`
+	} `json:"args,omitempty"`
+	RuntimeConfig struct {
+		Mac string `json:"mac,omitempty"`
+	} `json:"runtimeConfig,omitempty"`
+
+	mac string
+}
+
+type BridgeArgs struct {
+	Mac string `json:"mac,omitempty"`
+}
+
+// MacEnvArgs represents CNI_ARGS
+type MacEnvArgs struct {
+	types.CommonArgs
+	MAC types.UnmarshallableString `json:"mac,omitempty"`
 }
 
 type gwInfo struct {
@@ -70,7 +91,7 @@ func init() {
 	runtime.LockOSThread()
 }
 
-func loadNetConf(bytes []byte) (*NetConf, string, error) {
+func loadNetConf(bytes []byte, envArgs string) (*NetConf, string, error) {
 	n := &NetConf{
 		BrName: defaultBrName,
 	}
@@ -80,6 +101,26 @@ func loadNetConf(bytes []byte) (*NetConf, string, error) {
 	if n.Vlan < 0 || n.Vlan > 4094 {
 		return nil, "", fmt.Errorf("invalid VLAN ID %d (must be between 0 and 4094)", n.Vlan)
 	}
+
+	if envArgs != "" {
+		e := MacEnvArgs{}
+		if err := types.LoadArgs(envArgs, &e); err != nil {
+			return nil, "", err
+		}
+
+		if e.MAC != "" {
+			n.mac = string(e.MAC)
+		}
+	}
+
+	if mac := n.Args.Cni.Mac; mac != "" {
+		n.mac = mac
+	}
+
+	if mac := n.RuntimeConfig.Mac; mac != "" {
+		n.mac = mac
+	}
+
 	return n, n.CNIVersion, nil
 }
 
@@ -273,7 +314,7 @@ func ensureVlanInterface(br *netlink.Bridge, vlanId int) (netlink.Link, error) {
 			return nil, fmt.Errorf("faild to find host namespace: %v", err)
 		}
 
-		_, brGatewayIface, err := setupVeth(hostNS, br, name, br.MTU, false, vlanId)
+		_, brGatewayIface, err := setupVeth(hostNS, br, name, br.MTU, false, vlanId, "")
 		if err != nil {
 			return nil, fmt.Errorf("faild to create vlan gateway %q: %v", name, err)
 		}
@@ -282,18 +323,23 @@ func ensureVlanInterface(br *netlink.Bridge, vlanId int) (netlink.Link, error) {
 		if err != nil {
 			return nil, fmt.Errorf("failed to lookup %q: %v", brGatewayIface.Name, err)
 		}
+
+		err = netlink.LinkSetUp(brGatewayVeth)
+		if err != nil {
+			return nil, fmt.Errorf("failed to up %q: %v", brGatewayIface.Name, err)
+		}
 	}
 
 	return brGatewayVeth, nil
 }
 
-func setupVeth(netns ns.NetNS, br *netlink.Bridge, ifName string, mtu int, hairpinMode bool, vlanID int) (*current.Interface, *current.Interface, error) {
+func setupVeth(netns ns.NetNS, br *netlink.Bridge, ifName string, mtu int, hairpinMode bool, vlanID int, mac string) (*current.Interface, *current.Interface, error) {
 	contIface := &current.Interface{}
 	hostIface := &current.Interface{}
 
 	err := netns.Do(func(hostNS ns.NetNS) error {
 		// create the veth pair in the container and move host end into host netns
-		hostVeth, containerVeth, err := ip.SetupVeth(ifName, mtu, hostNS)
+		hostVeth, containerVeth, err := ip.SetupVeth(ifName, mtu, mac, hostNS)
 		if err != nil {
 			return err
 		}
@@ -356,20 +402,6 @@ func setupBridge(n *NetConf) (*netlink.Bridge, *current.Interface, error) {
 	}, nil
 }
 
-// disableIPV6DAD disables IPv6 Duplicate Address Detection (DAD)
-// for an interface, if the interface does not support enhanced_dad.
-// We do this because interfaces with hairpin mode will see their own DAD packets
-func disableIPV6DAD(ifName string) error {
-	// ehanced_dad sends a nonce with the DAD packets, so that we can safely
-	// ignore ourselves
-	enh, err := ioutil.ReadFile(fmt.Sprintf("/proc/sys/net/ipv6/conf/%s/enhanced_dad", ifName))
-	if err == nil && string(enh) == "1\n" {
-		return nil
-	}
-	f := fmt.Sprintf("/proc/sys/net/ipv6/conf/%s/accept_dad", ifName)
-	return ioutil.WriteFile(f, []byte("0"), 0644)
-}
-
 func enableIPForward(family int) error {
 	if family == netlink.FAMILY_V4 {
 		return ip.EnableIP4Forward()
@@ -380,7 +412,7 @@ func enableIPForward(family int) error {
 func cmdAdd(args *skel.CmdArgs) error {
 	var success bool = false
 
-	n, cniVersion, err := loadNetConf(args.StdinData)
+	n, cniVersion, err := loadNetConf(args.StdinData, args.Args)
 	if err != nil {
 		return err
 	}
@@ -392,7 +424,7 @@ func cmdAdd(args *skel.CmdArgs) error {
 	}
 
 	if n.HairpinMode && n.PromiscMode {
-		return fmt.Errorf("cannot set hairpin mode and promiscous mode at the same time.")
+		return fmt.Errorf("cannot set hairpin mode and promiscuous mode at the same time.")
 	}
 
 	br, brInterface, err := setupBridge(n)
@@ -406,13 +438,34 @@ func cmdAdd(args *skel.CmdArgs) error {
 	}
 	defer netns.Close()
 
-	hostInterface, containerInterface, err := setupVeth(netns, br, args.IfName, n.MTU, n.HairpinMode, n.Vlan)
+	hostInterface, containerInterface, err := setupVeth(netns, br, args.IfName, n.MTU, n.HairpinMode, n.Vlan, n.mac)
 	if err != nil {
 		return err
 	}
 
 	// Assume L2 interface only
-	result := &current.Result{CNIVersion: cniVersion, Interfaces: []*current.Interface{brInterface, hostInterface, containerInterface}}
+	result := &current.Result{
+		CNIVersion: current.ImplementedSpecVersion,
+		Interfaces: []*current.Interface{
+			brInterface,
+			hostInterface,
+			containerInterface,
+		},
+	}
+
+	if n.MacSpoofChk {
+		sc := link.NewSpoofChecker(hostInterface.Name, containerInterface.Mac, uniqueID(args.ContainerID, args.IfName))
+		if err := sc.Setup(); err != nil {
+			return err
+		}
+		defer func() {
+			if !success {
+				if err := sc.Teardown(); err != nil {
+					fmt.Fprintf(os.Stderr, "%v", err)
+				}
+			}
+		}()
+	}
 
 	if isLayer3 {
 		// run the IPAM plugin and get back the config to apply
@@ -449,17 +502,13 @@ func cmdAdd(args *skel.CmdArgs) error {
 
 		// Configure the container hardware address and IP address(es)
 		if err := netns.Do(func(_ ns.NetNS) error {
-			// Disable IPv6 DAD just in case hairpin mode is enabled on the
-			// bridge. Hairpin mode causes echos of neighbor solicitation
-			// packets, which causes DAD failures.
-			for _, ipc := range result.IPs {
-				if ipc.Version == "6" && (n.HairpinMode || n.PromiscMode) {
-					if err := disableIPV6DAD(args.IfName); err != nil {
-						return err
-					}
-					break
-				}
+			if n.EnableDad {
+				_, _ = sysctl.Sysctl(fmt.Sprintf("/net/ipv6/conf/%s/enhanced_dad", args.IfName), "1")
+				_, _ = sysctl.Sysctl(fmt.Sprintf("net/ipv6/conf/%s/accept_dad", args.IfName), "1")
+			} else {
+				_, _ = sysctl.Sysctl(fmt.Sprintf("net/ipv6/conf/%s/accept_dad", args.IfName), "0")
 			}
+			_, _ = sysctl.Sysctl(fmt.Sprintf("net/ipv4/conf/%s/arp_notify", args.IfName), "1")
 
 			// Add the IP to the interface
 			if err := ipam.ConfigureIface(args.IfName, result); err != nil {
@@ -488,23 +537,6 @@ func cmdAdd(args *skel.CmdArgs) error {
 			}
 		}
 
-		// Send a gratuitous arp
-		if err := netns.Do(func(_ ns.NetNS) error {
-			contVeth, err := net.InterfaceByName(args.IfName)
-			if err != nil {
-				return err
-			}
-
-			for _, ipc := range result.IPs {
-				if ipc.Version == "4" {
-					_ = arping.GratuitousArpOverIface(ipc.Address.IP, *contVeth)
-				}
-			}
-			return nil
-		}); err != nil {
-			return err
-		}
-
 		if n.IsGW {
 			var firstV4Addr net.IP
 			var vlanInterface *current.Interface
@@ -555,6 +587,20 @@ func cmdAdd(args *skel.CmdArgs) error {
 				}
 			}
 		}
+	} else {
+		if err := netns.Do(func(_ ns.NetNS) error {
+			link, err := netlink.LinkByName(args.IfName)
+			if err != nil {
+				return fmt.Errorf("failed to retrieve link: %v", err)
+			}
+			// If layer 2 we still need to set the container veth to up
+			if err = netlink.LinkSetUp(link); err != nil {
+				return fmt.Errorf("failed to set %q up: %v", args.IfName, err)
+			}
+			return nil
+		}); err != nil {
+			return err
+		}
 	}
 
 	// Refetch the bridge since its MAC address may change when the first
@@ -578,7 +624,7 @@ func cmdAdd(args *skel.CmdArgs) error {
 }
 
 func cmdDel(args *skel.CmdArgs) error {
-	n, _, err := loadNetConf(args.StdinData)
+	n, _, err := loadNetConf(args.StdinData, args.Args)
 	if err != nil {
 		return err
 	}
@@ -609,9 +655,23 @@ func cmdDel(args *skel.CmdArgs) error {
 	})
 
 	if err != nil {
+		//  if NetNs is passed down by the Cloud Orchestration Engine, or if it called multiple times
+		// so don't return an error if the device is already removed.
+		// https://github.com/kubernetes/kubernetes/issues/43014#issuecomment-287164444
+		_, ok := err.(ns.NSPathNotExistErr)
+		if ok {
+			return nil
+		}
 		return err
 	}
 
+	if n.MacSpoofChk {
+		sc := link.NewSpoofChecker("", "", uniqueID(args.ContainerID, args.IfName))
+		if err := sc.Teardown(); err != nil {
+			fmt.Fprintf(os.Stderr, "%v", err)
+		}
+	}
+
 	if isLayer3 && n.IPMasq {
 		chain := utils.FormatChainName(n.Name, args.ContainerID)
 		comment := utils.FormatComment(n.Name, args.ContainerID)
@@ -769,7 +829,7 @@ func validateCniContainerInterface(intf current.Interface) (cniBridgeIf, error)
 
 func cmdCheck(args *skel.CmdArgs) error {
 
-	n, _, err := loadNetConf(args.StdinData)
+	n, _, err := loadNetConf(args.StdinData, args.Args)
 	if err != nil {
 		return err
 	}
@@ -892,3 +952,7 @@ func cmdCheck(args *skel.CmdArgs) error {
 
 	return nil
 }
+
+func uniqueID(containerID, cniIface string) string {
+	return containerID + "-" + cniIface
+}

plugins/main/host-device/README.md

@@ -1,66 +1,5 @@
-# host-device
 
-Move an already-existing device into a container.
+This document has moved to the [containernetworking/cni.dev](https://github.com/containernetworking/cni.dev) repo.
 
-## Overview
+You can find it online here: https://cni.dev/plugins/current/main/host-device/
 
-This simple plugin will move the requested device from the host's network namespace
-to the container's. IPAM configuration can be used for this plugin.
-
-## Network configuration reference
-
-The device can be specified with any one of four properties:
-* `device`: The device name, e.g. `eth0`, `can0`
-* `hwaddr`: A MAC address
-* `kernelpath`: The kernel device kobj, e.g. `/sys/devices/pci0000:00/0000:00:1f.6`
-* `pciBusID`: A PCI address of network device, e.g `0000:00:1f.6`
-
-For this plugin, `CNI_IFNAME` will be ignored. Upon DEL, the device will be moved back.
-
-The plugin also supports the following [capability argument](https://github.com/containernetworking/cni/blob/master/CONVENTIONS.md):
-* `deviceID`: A PCI address of the network device, e.g `0000:00:1f.6`
-
-## Example configuration
-
-A sample configuration with `device` property looks like:
-
-```json
-{
-	"cniVersion": "0.3.1",
-	"type": "host-device",
-	"device": "enp0s1"
-}
-```
-
-A sample configuration with `pciBusID` property looks like:
-
-```json
-{
-	"cniVersion": "0.3.1",
-	"type": "host-device",
-	"pciBusID": "0000:3d:00.1"
-}
-```
-
-A sample configuration utilizing `deviceID` runtime configuration looks like:
-
-1. From operator perspective:
-     ```json
-    {
-    	"cniVersion": "0.3.1",
-    	"type": "host-device",
-    	"capabilities": {
-    		"deviceID":  true
-    	}
-    }
-    ```
-2. From plugin perspective:
-    ```json
-    {
-    	"cniVersion": "0.3.1",
-    	"type": "host-device",
-    	"runtimeConfig": {
-    		"deviceID":  "0000:3d:00.1"
-    	}
-    }
-    ```

plugins/main/host-device/host-device.go

@@ -30,7 +30,7 @@ import (
 
 	"github.com/containernetworking/cni/pkg/skel"
 	"github.com/containernetworking/cni/pkg/types"
-	"github.com/containernetworking/cni/pkg/types/current"
+	current "github.com/containernetworking/cni/pkg/types/100"
 	"github.com/containernetworking/cni/pkg/version"
 
 	"github.com/containernetworking/plugins/pkg/ip"
@@ -39,15 +39,19 @@ import (
 	bv "github.com/containernetworking/plugins/pkg/utils/buildversion"
 )
 
-const (
+var (
 	sysBusPCI = "/sys/bus/pci/devices"
 )
 
+// Array of different linux drivers bound to network device needed for DPDK
+var userspaceDrivers = []string{"vfio-pci", "uio_pci_generic", "igb_uio"}
+
 //NetConf for host-device config, look the README to learn how to use those parameters
 type NetConf struct {
 	types.NetConf
-	Device        string `json:"device"`     // Device-Name, something like eth0 or can0 etc.
-	HWAddr        string `json:"hwaddr"`     // MAC Address of target network interface
+	Device        string `json:"device"` // Device-Name, something like eth0 or can0 etc.
+	HWAddr        string `json:"hwaddr"` // MAC Address of target network interface
+	DPDKMode      bool
 	KernelPath    string `json:"kernelpath"` // Kernelpath of the device
 	PCIAddr       string `json:"pciBusID"`   // PCI Address of target network device
 	RuntimeConfig struct {
@@ -64,7 +68,8 @@ func init() {
 
 func loadConf(bytes []byte) (*NetConf, error) {
 	n := &NetConf{}
-	if err := json.Unmarshal(bytes, n); err != nil {
+	var err error
+	if err = json.Unmarshal(bytes, n); err != nil {
 		return nil, fmt.Errorf("failed to load netconf: %v", err)
 	}
 
@@ -77,6 +82,13 @@ func loadConf(bytes []byte) (*NetConf, error) {
 		return nil, fmt.Errorf(`specify either "device", "hwaddr", "kernelpath" or "pciBusID"`)
 	}
 
+	if len(n.PCIAddr) > 0 {
+		n.DPDKMode, err = hasDpdkDriver(n.PCIAddr)
+		if err != nil {
+			return nil, fmt.Errorf("error with host device: %v", err)
+		}
+	}
+
 	return n, nil
 }
 
@@ -91,39 +103,17 @@ func cmdAdd(args *skel.CmdArgs) error {
 	}
 	defer containerNs.Close()
 
-	hostDev, err := getLink(cfg.Device, cfg.HWAddr, cfg.KernelPath, cfg.PCIAddr)
-	if err != nil {
-		return fmt.Errorf("failed to find host device: %v", err)
-	}
-
-	contDev, err := moveLinkIn(hostDev, containerNs, args.IfName)
-	if err != nil {
-		return fmt.Errorf("failed to move link %v", err)
-	}
-
-	var result *current.Result
-	// run the IPAM plugin and get back the config to apply
-	if cfg.IPAM.Type != "" {
-		r, err := ipam.ExecAdd(cfg.IPAM.Type, args.StdinData)
+	result := &current.Result{}
+	var contDev netlink.Link
+	if !cfg.DPDKMode {
+		hostDev, err := getLink(cfg.Device, cfg.HWAddr, cfg.KernelPath, cfg.PCIAddr)
 		if err != nil {
-			return err
+			return fmt.Errorf("failed to find host device: %v", err)
 		}
 
-		// Invoke ipam del if err to avoid ip leak
-		defer func() {
-			if err != nil {
-				ipam.ExecDel(cfg.IPAM.Type, args.StdinData)
-			}
-		}()
-
-		// Convert whatever the IPAM result was into the current Result type
-		result, err = current.NewResultFromResult(r)
+		contDev, err = moveLinkIn(hostDev, containerNs, args.IfName)
 		if err != nil {
-			return err
-		}
-
-		if len(result.IPs) == 0 {
-			return errors.New("IPAM plugin returned missing IP config")
+			return fmt.Errorf("failed to move link %v", err)
 		}
 
 		result.Interfaces = []*current.Interface{{
@@ -131,13 +121,48 @@ func cmdAdd(args *skel.CmdArgs) error {
 			Mac:     contDev.Attrs().HardwareAddr.String(),
 			Sandbox: containerNs.Path(),
 		}}
-		for _, ipc := range result.IPs {
-			// All addresses apply to the container interface (move from host)
-			ipc.Interface = current.Int(0)
-		}
+	}
 
+	if cfg.IPAM.Type == "" {
+		if cfg.DPDKMode {
+			return types.PrintResult(result, cfg.CNIVersion)
+		}
+		return printLink(contDev, cfg.CNIVersion, containerNs)
+	}
+
+	// run the IPAM plugin and get back the config to apply
+	r, err := ipam.ExecAdd(cfg.IPAM.Type, args.StdinData)
+	if err != nil {
+		return err
+	}
+
+	// Invoke ipam del if err to avoid ip leak
+	defer func() {
+		if err != nil {
+			ipam.ExecDel(cfg.IPAM.Type, args.StdinData)
+		}
+	}()
+
+	// Convert whatever the IPAM result was into the current Result type
+	newResult, err := current.NewResultFromResult(r)
+	if err != nil {
+		return err
+	}
+
+	if len(newResult.IPs) == 0 {
+		return errors.New("IPAM plugin returned missing IP config")
+	}
+
+	for _, ipc := range newResult.IPs {
+		// All addresses apply to the container interface (move from host)
+		ipc.Interface = current.Int(0)
+	}
+
+	newResult.Interfaces = result.Interfaces
+
+	if !cfg.DPDKMode {
 		err = containerNs.Do(func(_ ns.NetNS) error {
-			if err := ipam.ConfigureIface(args.IfName, result); err != nil {
+			if err := ipam.ConfigureIface(args.IfName, newResult); err != nil {
 				return err
 			}
 			return nil
@@ -145,13 +170,11 @@ func cmdAdd(args *skel.CmdArgs) error {
 		if err != nil {
 			return err
 		}
-
-		result.DNS = cfg.DNS
-
-		return types.PrintResult(result, cfg.CNIVersion)
 	}
 
-	return printLink(contDev, cfg.CNIVersion, containerNs)
+	newResult.DNS = cfg.DNS
+
+	return types.PrintResult(newResult, cfg.CNIVersion)
 }
 
 func cmdDel(args *skel.CmdArgs) error {
@@ -168,16 +191,18 @@ func cmdDel(args *skel.CmdArgs) error {
 	}
 	defer containerNs.Close()
 
-	if err := moveLinkOut(containerNs, args.IfName); err != nil {
-		return err
-	}
-
 	if cfg.IPAM.Type != "" {
 		if err := ipam.ExecDel(cfg.IPAM.Type, args.StdinData); err != nil {
 			return err
 		}
 	}
 
+	if !cfg.DPDKMode {
+		if err := moveLinkOut(containerNs, args.IfName); err != nil {
+			return err
+		}
+	}
+
 	return nil
 }
 
@@ -205,6 +230,10 @@ func moveLinkIn(hostDev netlink.Link, containerNs ns.NetNS, ifName string) (netl
 		if err := netlink.LinkSetName(contDev, ifName); err != nil {
 			return fmt.Errorf("failed to rename device %q to %q: %v", hostDev.Attrs().Name, ifName, err)
 		}
+		// Bring container device up
+		if err = netlink.LinkSetUp(contDev); err != nil {
+			return fmt.Errorf("failed to set %q up: %v", ifName, err)
+		}
 		// Retrieve link again to get up-to-date name and attributes
 		contDev, err = netlink.LinkByName(ifName)
 		if err != nil {
@@ -236,17 +265,21 @@ func moveLinkOut(containerNs ns.NetNS, ifName string) error {
 			return fmt.Errorf("failed to set %q down: %v", ifName, err)
 		}
 
-		// Rename device to it's original name
+		defer func() {
+			// If moving the device to the host namespace fails, set its name back to ifName so that this
+			// function can be retried. Also bring the device back up, unless it was already down before.
+			if err != nil {
+				_ = netlink.LinkSetName(dev, ifName)
+				if dev.Attrs().Flags&net.FlagUp == net.FlagUp {
+					_ = netlink.LinkSetUp(dev)
+				}
+			}
+		}()
+
+		// Rename the device to its original name from the host namespace
 		if err = netlink.LinkSetName(dev, dev.Attrs().Alias); err != nil {
 			return fmt.Errorf("failed to restore %q to original name %q: %v", ifName, dev.Attrs().Alias, err)
 		}
-		defer func() {
-			if err != nil {
-				// if moving device to host namespace fails, we should revert device name
-				// to ifName to make sure that device can be found in retries
-				_ = netlink.LinkSetName(dev, ifName)
-			}
-		}()
 
 		if err = netlink.LinkSetNsFd(dev, int(defaultNs.Fd())); err != nil {
 			return fmt.Errorf("failed to move %q to host netns: %v", dev.Attrs().Alias, err)
@@ -255,6 +288,25 @@ func moveLinkOut(containerNs ns.NetNS, ifName string) error {
 	})
 }
 
+func hasDpdkDriver(pciaddr string) (bool, error) {
+	driverLink := filepath.Join(sysBusPCI, pciaddr, "driver")
+	driverPath, err := filepath.EvalSymlinks(driverLink)
+	if err != nil {
+		return false, err
+	}
+	driverStat, err := os.Stat(driverPath)
+	if err != nil {
+		return false, err
+	}
+	driverName := driverStat.Name()
+	for _, drv := range userspaceDrivers {
+		if driverName == drv {
+			return true, nil
+		}
+	}
+	return false, nil
+}
+
 func printLink(dev netlink.Link, cniVersion string, containerNs ns.NetNS) error {
 	result := current.Result{
 		CNIVersion: current.ImplementedSpecVersion,
@@ -368,6 +420,10 @@ func cmdCheck(args *skel.CmdArgs) error {
 		return err
 	}
 
+	if cfg.DPDKMode {
+		return nil
+	}
+
 	var contMap current.Interface
 	// Find interfaces for name we know, that of host-device inside container
 	for _, intf := range result.Interfaces {

plugins/main/ipvlan/README.md

@@ -1,45 +1,5 @@
-# ipvlan plugin
 
-## Overview
+This document has moved to the [containernetworking/cni.dev](https://github.com/containernetworking/cni.dev) repo.
 
-ipvlan is a new [addition](https://lwn.net/Articles/620087/) to the Linux kernel.
-Like its cousin macvlan, it virtualizes the host interface.
-However unlike macvlan which generates a new MAC address for each interface, ipvlan devices all share the same MAC.
-The kernel driver inspects the IP address of each packet when making a decision about which virtual interface should process the packet.
+You can find it online here: https://cni.dev/plugins/current/main/ipvlan/
 
-Because all ipvlan interfaces share the MAC address with the host interface, DHCP can only be used in conjunction with ClientID (currently not supported by DHCP plugin).
-
-## Example configuration
-
-```
-{
-	"name": "mynet",
-	"type": "ipvlan",
-	"master": "eth0",
-	"ipam": {
-		"type": "host-local",
-		"subnet": "10.1.2.0/24"
-	}
-}
-```
-
-## Network configuration reference
-
-* `name` (string, required): the name of the network.
-* `type` (string, required): "ipvlan".
-* `master` (string, required unless chained): name of the host interface to enslave.
-* `mode` (string, optional): one of "l2", "l3", "l3s". Defaults to "l2".
-* `mtu` (integer, optional): explicitly set MTU to the specified value. Defaults to the value chosen by the kernel.
-* `ipam` (dictionary, required unless chained): IPAM configuration to be used for this network.
-
-## Notes
-
-* `ipvlan` does not allow virtual interfaces to communicate with the master interface.
-Therefore the container will not be able to reach the host via `ipvlan` interface.
-Be sure to also have container join a network that provides connectivity to the host (e.g. `ptp`).
-* A single master interface can not be enslaved by both `macvlan` and `ipvlan`.
-* For IP allocation schemes that cannot be interface agnostic, the ipvlan plugin
-can be chained with an earlier plugin that handles this logic. If `master` is
-omitted, then the previous Result must contain a single interface name for the
-ipvlan plugin to enslave. If `ipam` is omitted, then the previous Result is used
-to configure the ipvlan interface.

plugins/main/ipvlan/ipvlan.go

@@ -24,13 +24,14 @@ import (
 
 	"github.com/containernetworking/cni/pkg/skel"
 	"github.com/containernetworking/cni/pkg/types"
-	"github.com/containernetworking/cni/pkg/types/current"
+	current "github.com/containernetworking/cni/pkg/types/100"
 	"github.com/containernetworking/cni/pkg/version"
 
 	"github.com/containernetworking/plugins/pkg/ip"
 	"github.com/containernetworking/plugins/pkg/ipam"
 	"github.com/containernetworking/plugins/pkg/ns"
 	bv "github.com/containernetworking/plugins/pkg/utils/buildversion"
+	"github.com/containernetworking/plugins/pkg/utils/sysctl"
 )
 
 type NetConf struct {
@@ -72,12 +73,18 @@ func loadConf(bytes []byte, cmdCheck bool) (*NetConf, string, error) {
 	}
 	if n.Master == "" {
 		if result == nil {
-			return nil, "", fmt.Errorf(`"master" field is required. It specifies the host interface name to virtualize`)
-		}
-		if len(result.Interfaces) == 1 && result.Interfaces[0].Name != "" {
-			n.Master = result.Interfaces[0].Name
+			defaultRouteInterface, err := getDefaultRouteInterfaceName()
+
+			if err != nil {
+				return nil, "", err
+			}
+			n.Master = defaultRouteInterface
 		} else {
-			return nil, "", fmt.Errorf("chained master failure. PrevResult lacks a single named interface")
+			if len(result.Interfaces) == 1 && result.Interfaces[0].Name != "" {
+				n.Master = result.Interfaces[0].Name
+			} else {
+				return nil, "", fmt.Errorf("chained master failure. PrevResult lacks a single named interface")
+			}
 		}
 	}
 	return n, n.CNIVersion, nil
@@ -167,6 +174,25 @@ func createIpvlan(conf *NetConf, ifName string, netns ns.NetNS) (*current.Interf
 	return ipvlan, nil
 }
 
+func getDefaultRouteInterfaceName() (string, error) {
+	routeToDstIP, err := netlink.RouteList(nil, netlink.FAMILY_ALL)
+	if err != nil {
+		return "", err
+	}
+
+	for _, v := range routeToDstIP {
+		if v.Dst == nil {
+			l, err := netlink.LinkByIndex(v.LinkIndex)
+			if err != nil {
+				return "", err
+			}
+			return l.Attrs().Name, nil
+		}
+	}
+
+	return "", fmt.Errorf("no default route interface found")
+}
+
 func cmdAdd(args *skel.CmdArgs) error {
 	n, cniVersion, err := loadConf(args.StdinData, false)
 	if err != nil {
@@ -229,7 +255,12 @@ func cmdAdd(args *skel.CmdArgs) error {
 	result.Interfaces = []*current.Interface{ipvlanInterface}
 
 	err = netns.Do(func(_ ns.NetNS) error {
-		return ipam.ConfigureIface(args.IfName, result)
+		_, _ = sysctl.Sysctl(fmt.Sprintf("net/ipv4/conf/%s/arp_notify", args.IfName), "1")
+
+		if err := ipam.ConfigureIface(args.IfName, result); err != nil {
+			return err
+		}
+		return nil
 	})
 	if err != nil {
 		return err
@@ -269,6 +300,17 @@ func cmdDel(args *skel.CmdArgs) error {
 		return nil
 	})
 
+	if err != nil {
+		//  if NetNs is passed down by the Cloud Orchestration Engine, or if it called multiple times
+		// so don't return an error if the device is already removed.
+		// https://github.com/kubernetes/kubernetes/issues/43014#issuecomment-287164444
+		_, ok := err.(ns.NSPathNotExistErr)
+		if ok {
+			return nil
+		}
+		return err
+	}
+
 	return err
 }
 

plugins/main/ipvlan/ipvlan_test.go

@@ -17,12 +17,17 @@ package main
 import (
 	"encoding/json"
 	"fmt"
+	"io/ioutil"
 	"net"
+	"os"
+	"strings"
 	"syscall"
 
 	"github.com/containernetworking/cni/pkg/skel"
 	"github.com/containernetworking/cni/pkg/types"
-	"github.com/containernetworking/cni/pkg/types/current"
+	"github.com/containernetworking/cni/pkg/types/020"
+	"github.com/containernetworking/cni/pkg/types/040"
+	"github.com/containernetworking/cni/pkg/types/100"
 	"github.com/containernetworking/plugins/pkg/ns"
 	"github.com/containernetworking/plugins/pkg/testutils"
 
@@ -44,37 +49,33 @@ type Net struct {
 	IPAM          *allocator.IPAMConfig  `json:"ipam"`
 	DNS           types.DNS              `json:"dns"`
 	RawPrevResult map[string]interface{} `json:"prevResult,omitempty"`
-	PrevResult    current.Result         `json:"-"`
+	PrevResult    types100.Result        `json:"-"`
 }
 
-func buildOneConfig(netName string, cniVersion string, master string, orig *Net, prevResult types.Result) (*Net, error) {
-	var err error
-
-	inject := map[string]interface{}{
-		"name":       netName,
-		"cniVersion": cniVersion,
-	}
-	// Add previous plugin result
-	if prevResult != nil {
-		inject["prevResult"] = prevResult
-	}
-	if orig.IPAM == nil {
-		inject["master"] = master
-	}
-
-	// Ensure every config uses the same name and version
-	config := make(map[string]interface{})
-
+func buildOneConfig(cniVersion string, master string, orig *Net, prevResult types.Result) (*Net, error) {
 	confBytes, err := json.Marshal(orig)
 	if err != nil {
 		return nil, err
 	}
 
+	config := make(map[string]interface{})
 	err = json.Unmarshal(confBytes, &config)
 	if err != nil {
 		return nil, fmt.Errorf("unmarshal existing network bytes: %s", err)
 	}
 
+	inject := map[string]interface{}{
+		"name":       orig.Name,
+		"cniVersion": orig.CNIVersion,
+	}
+	// Add previous plugin result
+	if prevResult != nil && testutils.SpecVersionHasChaining(cniVersion) {
+		inject["prevResult"] = prevResult
+	}
+	if master != "" {
+		inject["master"] = master
+	}
+
 	for key, value := range inject {
 		config[key] = value
 	}
@@ -93,121 +94,49 @@ func buildOneConfig(netName string, cniVersion string, master string, orig *Net,
 
 }
 
-func ipvlanAddDelTest(conf, IFNAME string, originalNS ns.NetNS) {
-	targetNs, err := testutils.NewNS()
+func ipvlanAddCheckDelTest(conf, masterName string, originalNS, targetNS ns.NetNS) {
+	// Unmarshal to pull out CNI spec version
+	rawConfig := make(map[string]interface{})
+	err := json.Unmarshal([]byte(conf), &rawConfig)
 	Expect(err).NotTo(HaveOccurred())
-	defer targetNs.Close()
+	cniVersion := rawConfig["cniVersion"].(string)
 
 	args := &skel.CmdArgs{
 		ContainerID: "dummy",
-		Netns:       targetNs.Path(),
-		IfName:      IFNAME,
+		Netns:       targetNS.Path(),
+		IfName:      "ipvl0",
 		StdinData:   []byte(conf),
 	}
 
-	var result *current.Result
+	var result types.Result
+	var macAddress string
 	err = originalNS.Do(func(ns.NetNS) error {
 		defer GinkgoRecover()
 
-		r, _, err := testutils.CmdAddWithArgs(args, func() error {
+		result, _, err = testutils.CmdAddWithArgs(args, func() error {
 			return cmdAdd(args)
 		})
 		Expect(err).NotTo(HaveOccurred())
 
-		result, err = current.GetResult(r)
-		Expect(err).NotTo(HaveOccurred())
-
-		Expect(len(result.Interfaces)).To(Equal(1))
-		Expect(result.Interfaces[0].Name).To(Equal(IFNAME))
-		Expect(len(result.IPs)).To(Equal(1))
+		t := newTesterByVersion(cniVersion)
+		macAddress = t.verifyResult(result, args.IfName)
 		return nil
 	})
 	Expect(err).NotTo(HaveOccurred())
 
 	// Make sure ipvlan link exists in the target namespace
-	err = targetNs.Do(func(ns.NetNS) error {
+	err = targetNS.Do(func(ns.NetNS) error {
 		defer GinkgoRecover()
 
-		link, err := netlink.LinkByName(IFNAME)
+		link, err := netlink.LinkByName(args.IfName)
 		Expect(err).NotTo(HaveOccurred())
-		Expect(link.Attrs().Name).To(Equal(IFNAME))
+		Expect(link.Attrs().Name).To(Equal(args.IfName))
 
-		hwaddr, err := net.ParseMAC(result.Interfaces[0].Mac)
-		Expect(err).NotTo(HaveOccurred())
-		Expect(link.Attrs().HardwareAddr).To(Equal(hwaddr))
-
-		addrs, err := netlink.AddrList(link, syscall.AF_INET)
-		Expect(err).NotTo(HaveOccurred())
-		Expect(len(addrs)).To(Equal(1))
-		return nil
-	})
-	Expect(err).NotTo(HaveOccurred())
-
-	err = originalNS.Do(func(ns.NetNS) error {
-		defer GinkgoRecover()
-
-		err = testutils.CmdDelWithArgs(args, func() error {
-			return cmdDel(args)
-		})
-		Expect(err).NotTo(HaveOccurred())
-		return nil
-	})
-	Expect(err).NotTo(HaveOccurred())
-
-	// Make sure ipvlan link has been deleted
-	err = targetNs.Do(func(ns.NetNS) error {
-		defer GinkgoRecover()
-
-		link, err := netlink.LinkByName(IFNAME)
-		Expect(err).To(HaveOccurred())
-		Expect(link).To(BeNil())
-		return nil
-	})
-	Expect(err).NotTo(HaveOccurred())
-}
-
-func ipvlanAddCheckDelTest(conf string, netName string, IFNAME string, originalNS ns.NetNS) {
-	targetNs, err := testutils.NewNS()
-	Expect(err).NotTo(HaveOccurred())
-	defer targetNs.Close()
-
-	args := &skel.CmdArgs{
-		ContainerID: "dummy",
-		Netns:       targetNs.Path(),
-		IfName:      IFNAME,
-		StdinData:   []byte(conf),
-	}
-
-	var result *current.Result
-	err = originalNS.Do(func(ns.NetNS) error {
-		defer GinkgoRecover()
-
-		r, _, err := testutils.CmdAddWithArgs(args, func() error {
-			return cmdAdd(args)
-		})
-		Expect(err).NotTo(HaveOccurred())
-
-		result, err = current.GetResult(r)
-		Expect(err).NotTo(HaveOccurred())
-
-		Expect(len(result.Interfaces)).To(Equal(1))
-		Expect(result.Interfaces[0].Name).To(Equal(IFNAME))
-		Expect(len(result.IPs)).To(Equal(1))
-		return nil
-	})
-	Expect(err).NotTo(HaveOccurred())
-
-	// Make sure ipvlan link exists in the target namespace
-	err = targetNs.Do(func(ns.NetNS) error {
-		defer GinkgoRecover()
-
-		link, err := netlink.LinkByName(IFNAME)
-		Expect(err).NotTo(HaveOccurred())
-		Expect(link.Attrs().Name).To(Equal(IFNAME))
-
-		hwaddr, err := net.ParseMAC(result.Interfaces[0].Mac)
-		Expect(err).NotTo(HaveOccurred())
-		Expect(link.Attrs().HardwareAddr).To(Equal(hwaddr))
+		if macAddress != "" {
+			hwaddr, err := net.ParseMAC(macAddress)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(link.Attrs().HardwareAddr).To(Equal(hwaddr))
+		}
 
 		addrs, err := netlink.AddrList(link, syscall.AF_INET)
 		Expect(err).NotTo(HaveOccurred())
@@ -225,26 +154,26 @@ func ipvlanAddCheckDelTest(conf string, netName string, IFNAME string, originalN
 		Expect(err).NotTo(HaveOccurred())
 	}
 
-	cniVersion := "0.4.0"
-	newConf, err := buildOneConfig(netName, cniVersion, MASTER_NAME, n, result)
+	// build chained/cached config for DEL
+	newConf, err := buildOneConfig(cniVersion, masterName, n, result)
+	Expect(err).NotTo(HaveOccurred())
+	confBytes, err := json.Marshal(newConf)
 	Expect(err).NotTo(HaveOccurred())
 
-	confString, err := json.Marshal(newConf)
-	Expect(err).NotTo(HaveOccurred())
+	args.StdinData = confBytes
+	GinkgoT().Logf(string(confBytes))
 
-	args.StdinData = confString
+	if testutils.SpecVersionHasCHECK(cniVersion) {
+		// CNI Check on ipvlan in the target namespace
+		err = originalNS.Do(func(ns.NetNS) error {
+			defer GinkgoRecover()
 
-	// CNI Check on macvlan in the target namespace
-	err = originalNS.Do(func(ns.NetNS) error {
-		defer GinkgoRecover()
-
-		err := testutils.CmdCheckWithArgs(args, func() error {
-			return cmdCheck(args)
+			return testutils.CmdCheckWithArgs(args, func() error {
+				return cmdCheck(args)
+			})
 		})
 		Expect(err).NotTo(HaveOccurred())
-		return nil
-	})
-	Expect(err).NotTo(HaveOccurred())
+	}
 
 	err = originalNS.Do(func(ns.NetNS) error {
 		defer GinkgoRecover()
@@ -258,10 +187,10 @@ func ipvlanAddCheckDelTest(conf string, netName string, IFNAME string, originalN
 	Expect(err).NotTo(HaveOccurred())
 
 	// Make sure ipvlan link has been deleted
-	err = targetNs.Do(func(ns.NetNS) error {
+	err = targetNS.Do(func(ns.NetNS) error {
 		defer GinkgoRecover()
 
-		link, err := netlink.LinkByName(IFNAME)
+		link, err := netlink.LinkByName(args.IfName)
 		Expect(err).To(HaveOccurred())
 		Expect(link).To(BeNil())
 		return nil
@@ -269,8 +198,70 @@ func ipvlanAddCheckDelTest(conf string, netName string, IFNAME string, originalN
 	Expect(err).NotTo(HaveOccurred())
 }
 
+type tester interface {
+	// verifyResult minimally verifies the Result and returns the interface's MAC address
+	verifyResult(result types.Result, name string) string
+}
+
+type testerBase struct{}
+
+type testerV10x testerBase
+type testerV04x testerBase
+type testerV02x testerBase
+
+func newTesterByVersion(version string) tester {
+	switch {
+	case strings.HasPrefix(version, "1.0."):
+		return &testerV10x{}
+	case strings.HasPrefix(version, "0.4.") || strings.HasPrefix(version, "0.3."):
+		return &testerV04x{}
+	case strings.HasPrefix(version, "0.1.") || strings.HasPrefix(version, "0.2."):
+		return &testerV02x{}
+	}
+	Fail(fmt.Sprintf("unsupported config version %s", version))
+	return nil
+}
+
+// verifyResult minimally verifies the Result and returns the interface's MAC address
+func (t *testerV10x) verifyResult(result types.Result, name string) string {
+	r, err := types100.GetResult(result)
+	Expect(err).NotTo(HaveOccurred())
+
+	Expect(len(r.Interfaces)).To(Equal(1))
+	Expect(r.Interfaces[0].Name).To(Equal(name))
+	Expect(len(r.IPs)).To(Equal(1))
+
+	return r.Interfaces[0].Mac
+}
+
+// verifyResult minimally verifies the Result and returns the interface's MAC address
+func (t *testerV04x) verifyResult(result types.Result, name string) string {
+	r, err := types040.GetResult(result)
+	Expect(err).NotTo(HaveOccurred())
+
+	Expect(len(r.Interfaces)).To(Equal(1))
+	Expect(r.Interfaces[0].Name).To(Equal(name))
+	Expect(len(r.IPs)).To(Equal(1))
+
+	return r.Interfaces[0].Mac
+}
+
+// verifyResult minimally verifies the Result and returns the interface's MAC address
+func (t *testerV02x) verifyResult(result types.Result, name string) string {
+	r, err := types020.GetResult(result)
+	Expect(err).NotTo(HaveOccurred())
+
+	Expect(r.IP4.IP).NotTo(BeNil())
+	Expect(r.IP4.IP.IP).NotTo(BeNil())
+	Expect(r.IP6).To(BeNil())
+
+	// 0.2 and earlier don't return MAC address
+	return ""
+}
+
 var _ = Describe("ipvlan Operations", func() {
-	var originalNS ns.NetNS
+	var originalNS, targetNS ns.NetNS
+	var dataDir string
 
 	BeforeEach(func() {
 		// Create a new NetNS so we don't modify the host
@@ -278,6 +269,12 @@ var _ = Describe("ipvlan Operations", func() {
 		originalNS, err = testutils.NewNS()
 		Expect(err).NotTo(HaveOccurred())
 
+		targetNS, err = testutils.NewNS()
+		Expect(err).NotTo(HaveOccurred())
+
+		dataDir, err = ioutil.TempDir("", "ipvlan_test")
+		Expect(err).NotTo(HaveOccurred())
+
 		err = originalNS.Do(func(ns.NetNS) error {
 			defer GinkgoRecover()
 
@@ -296,173 +293,170 @@ var _ = Describe("ipvlan Operations", func() {
 	})
 
 	AfterEach(func() {
+		Expect(os.RemoveAll(dataDir)).To(Succeed())
+
 		Expect(originalNS.Close()).To(Succeed())
 		Expect(testutils.UnmountNS(originalNS)).To(Succeed())
+
+		Expect(targetNS.Close()).To(Succeed())
+		Expect(testutils.UnmountNS(targetNS)).To(Succeed())
 	})
 
-	It("creates an ipvlan link in a non-default namespace", func() {
-		conf := &NetConf{
-			NetConf: types.NetConf{
-				CNIVersion: "0.3.1",
-				Name:       "testConfig",
-				Type:       "ipvlan",
-			},
-			Master: MASTER_NAME,
-			Mode:   "l2",
-			MTU:    1500,
-		}
+	for _, ver := range testutils.AllSpecVersions {
+		// Redefine ver inside for scope so real value is picked up by each dynamically defined It()
+		// See Gingkgo's "Patterns for dynamically generating tests" documentation.
+		ver := ver
 
-		// Create ipvlan in other namespace
-		targetNs, err := testutils.NewNS()
-		Expect(err).NotTo(HaveOccurred())
-		defer targetNs.Close()
+		It(fmt.Sprintf("[%s] creates an ipvlan link in a non-default namespace", ver), func() {
+			conf := &NetConf{
+				NetConf: types.NetConf{
+					CNIVersion: ver,
+					Name:       "testConfig",
+					Type:       "ipvlan",
+				},
+				Master: MASTER_NAME,
+				Mode:   "l2",
+				MTU:    1500,
+			}
 
-		err = originalNS.Do(func(ns.NetNS) error {
-			defer GinkgoRecover()
+			err := originalNS.Do(func(ns.NetNS) error {
+				defer GinkgoRecover()
+
+				_, err := createIpvlan(conf, "foobar0", targetNS)
+				Expect(err).NotTo(HaveOccurred())
+				return nil
+			})
 
-			_, err := createIpvlan(conf, "foobar0", targetNs)
 			Expect(err).NotTo(HaveOccurred())
-			return nil
-		})
 
-		Expect(err).NotTo(HaveOccurred())
+			// Make sure ipvlan link exists in the target namespace
+			err = targetNS.Do(func(ns.NetNS) error {
+				defer GinkgoRecover()
 
-		// Make sure ipvlan link exists in the target namespace
-		err = targetNs.Do(func(ns.NetNS) error {
-			defer GinkgoRecover()
-
-			link, err := netlink.LinkByName("foobar0")
-			Expect(err).NotTo(HaveOccurred())
-			Expect(link.Attrs().Name).To(Equal("foobar0"))
-			return nil
-		})
-		Expect(err).NotTo(HaveOccurred())
-	})
-
-	It("configures and deconfigures an iplvan link with ADD/DEL", func() {
-		const IFNAME = "ipvl0"
-
-		conf := fmt.Sprintf(`{
-    "cniVersion": "0.3.1",
-    "name": "mynet",
-    "type": "ipvlan",
-    "master": "%s",
-    "ipam": {
-        "type": "host-local",
-        "subnet": "10.1.2.0/24"
-    }
-}`, MASTER_NAME)
-
-		ipvlanAddDelTest(conf, IFNAME, originalNS)
-	})
-
-	It("configures and deconfigures an iplvan link with ADD/DEL when chained", func() {
-		const IFNAME = "ipvl0"
-
-		conf := fmt.Sprintf(`{
-    "cniVersion": "0.3.1",
-    "name": "mynet",
-    "type": "ipvlan",
-    "prevResult": {
-            "interfaces": [
-                    {
-                            "name": "%s"
-                    }
-            ],
-            "ips": [
-                    {
-                            "version": "4",
-                            "address": "10.1.2.2/24",
-                            "gateway": "10.1.2.1",
-                            "interface": 0
-                    }
-            ],
-            "routes": []
-    }
-}`, MASTER_NAME)
-
-		ipvlanAddDelTest(conf, IFNAME, originalNS)
-	})
-
-	It("deconfigures an unconfigured ipvlan link with DEL", func() {
-		const IFNAME = "ipvl0"
-
-		conf := fmt.Sprintf(`{
-    "cniVersion": "0.3.0",
-    "name": "mynet",
-    "type": "ipvlan",
-    "master": "%s",
-    "ipam": {
-        "type": "host-local",
-        "subnet": "10.1.2.0/24"
-    }
-}`, MASTER_NAME)
-
-		targetNs, err := testutils.NewNS()
-		Expect(err).NotTo(HaveOccurred())
-		defer targetNs.Close()
-
-		args := &skel.CmdArgs{
-			ContainerID: "dummy",
-			Netns:       targetNs.Path(),
-			IfName:      IFNAME,
-			StdinData:   []byte(conf),
-		}
-
-		err = originalNS.Do(func(ns.NetNS) error {
-			defer GinkgoRecover()
-
-			err = testutils.CmdDelWithArgs(args, func() error {
-				return cmdDel(args)
+				link, err := netlink.LinkByName("foobar0")
+				Expect(err).NotTo(HaveOccurred())
+				Expect(link.Attrs().Name).To(Equal("foobar0"))
+				return nil
 			})
 			Expect(err).NotTo(HaveOccurred())
-			return nil
 		})
-		Expect(err).NotTo(HaveOccurred())
-	})
 
-	It("configures and deconfigures a cniVersion 0.4.0 iplvan link with ADD/CHECK/DEL", func() {
-		const IFNAME = "ipvl0"
+		It(fmt.Sprintf("[%s] configures and deconfigures an iplvan link with ADD/DEL", ver), func() {
+			conf := fmt.Sprintf(`{
+			    "cniVersion": "%s",
+			    "name": "mynet",
+			    "type": "ipvlan",
+			    "master": "%s",
+			    "ipam": {
+				"type": "host-local",
+				"subnet": "10.1.2.0/24",
+				"dataDir": "%s"
+			    }
+			}`, ver, MASTER_NAME, dataDir)
 
-		conf := fmt.Sprintf(`{
-    "cniVersion": "0.4.0",
-    "name": "ipvlanTest1",
-    "type": "ipvlan",
-    "master": "%s",
-    "ipam": {
-        "type": "host-local",
-        "subnet": "10.1.2.0/24"
-    }
-}`, MASTER_NAME)
+			ipvlanAddCheckDelTest(conf, "", originalNS, targetNS)
+		})
 
-		ipvlanAddCheckDelTest(conf, "ipvlanTest1", IFNAME, originalNS)
-	})
+		if testutils.SpecVersionHasChaining(ver) {
+			It(fmt.Sprintf("[%s] configures and deconfigures an iplvan link with ADD/DEL when chained", ver), func() {
+				conf := fmt.Sprintf(`{
+				    "cniVersion": "%s",
+				    "name": "mynet",
+				    "type": "ipvlan",
+				    "prevResult": {
+					    "interfaces": [
+						    {
+							    "name": "%s"
+						    }
+					    ],
+					    "ips": [
+						    {
+							    "version": "4",
+							    "address": "10.1.2.2/24",
+							    "gateway": "10.1.2.1",
+							    "interface": 0
+						    }
+					    ],
+					    "routes": []
+				    }
+				}`, ver, MASTER_NAME)
 
-	It("configures and deconfigures a cniVersion 0.4.0 iplvan link with ADD/CHECK/DEL when chained", func() {
-		const IFNAME = "ipvl0"
+				ipvlanAddCheckDelTest(conf, MASTER_NAME, originalNS, targetNS)
+			})
+		}
 
-		conf := fmt.Sprintf(`{
-	       "cniVersion": "0.4.0",
-	       "name": "ipvlanTest2",
-	       "type": "ipvlan",
-	       "prevResult": {
-	               "interfaces": [
-	                       {
-	                               "name": "%s"
-	                       }
-	               ],
-	               "ips": [
-	                       {
-	                               "version": "4",
-	                               "address": "10.1.2.2/24",
-	                               "gateway": "10.1.2.1",
-	                               "interface": 0
-	                       }
-	               ],
-	               "routes": []
-	       }
-	   }`, MASTER_NAME)
+		It(fmt.Sprintf("[%s] deconfigures an unconfigured ipvlan link with DEL", ver), func() {
+			conf := fmt.Sprintf(`{
+			    "cniVersion": "%s",
+			    "name": "mynet",
+			    "type": "ipvlan",
+			    "master": "%s",
+			    "ipam": {
+				"type": "host-local",
+				"subnet": "10.1.2.0/24",
+				"dataDir": "%s"
+			    }
+			}`, ver, MASTER_NAME, dataDir)
 
-		ipvlanAddCheckDelTest(conf, "ipvlanTest2", IFNAME, originalNS)
-	})
+			args := &skel.CmdArgs{
+				ContainerID: "dummy",
+				Netns:       targetNS.Path(),
+				IfName:      "ipvl0",
+				StdinData:   []byte(conf),
+			}
+
+			err := originalNS.Do(func(ns.NetNS) error {
+				defer GinkgoRecover()
+
+				err := testutils.CmdDelWithArgs(args, func() error {
+					return cmdDel(args)
+				})
+				Expect(err).NotTo(HaveOccurred())
+				return nil
+			})
+			Expect(err).NotTo(HaveOccurred())
+		})
+
+		It(fmt.Sprintf("[%s] configures and deconfigures a ipvlan link with ADD/DEL, without master config", ver), func() {
+			conf := fmt.Sprintf(`{
+			    "cniVersion": "%s",
+			    "name": "mynet",
+			    "type": "ipvlan",
+			    "ipam": {
+				"type": "host-local",
+				"subnet": "10.1.2.0/24",
+				"dataDir": "%s"
+			    }
+			}`, ver, dataDir)
+
+			// Make MASTER_NAME as default route interface
+			err := originalNS.Do(func(ns.NetNS) error {
+				defer GinkgoRecover()
+
+				link, err := netlink.LinkByName(MASTER_NAME)
+				Expect(err).NotTo(HaveOccurred())
+				err = netlink.LinkSetUp(link)
+				Expect(err).NotTo(HaveOccurred())
+
+				var address = &net.IPNet{IP: net.IPv4(192, 0, 0, 1), Mask: net.CIDRMask(24, 32)}
+				var addr = &netlink.Addr{IPNet: address}
+				err = netlink.AddrAdd(link, addr)
+				Expect(err).NotTo(HaveOccurred())
+
+				// add default gateway into MASTER
+				dst := &net.IPNet{
+					IP:   net.IPv4(0, 0, 0, 0),
+					Mask: net.CIDRMask(0, 0),
+				}
+				ip := net.IPv4(192, 0, 0, 254)
+				route := netlink.Route{LinkIndex: link.Attrs().Index, Dst: dst, Gw: ip}
+				err = netlink.RouteAdd(&route)
+				Expect(err).NotTo(HaveOccurred())
+				return nil
+			})
+			Expect(err).NotTo(HaveOccurred())
+
+			ipvlanAddCheckDelTest(conf, MASTER_NAME, originalNS, targetNS)
+		})
+	}
 })

plugins/main/loopback/loopback.go

@@ -24,7 +24,7 @@ import (
 
 	"github.com/containernetworking/cni/pkg/skel"
 	"github.com/containernetworking/cni/pkg/types"
-	"github.com/containernetworking/cni/pkg/types/current"
+	current "github.com/containernetworking/cni/pkg/types/100"
 	"github.com/containernetworking/cni/pkg/version"
 
 	"github.com/containernetworking/plugins/pkg/ns"
@@ -109,12 +109,19 @@ func cmdAdd(args *skel.CmdArgs) error {
 		// loopback should pass it transparently
 		result = conf.PrevResult
 	} else {
-		loopbackInterface := &current.Interface{Name: args.IfName, Mac: "00:00:00:00:00:00", Sandbox: args.Netns}
-		r := &current.Result{CNIVersion: conf.CNIVersion, Interfaces: []*current.Interface{loopbackInterface}}
+		r := &current.Result{
+			CNIVersion: conf.CNIVersion,
+			Interfaces: []*current.Interface{
+				&current.Interface{
+					Name:    args.IfName,
+					Mac:     "00:00:00:00:00:00",
+					Sandbox: args.Netns,
+				},
+			},
+		}
 
 		if v4Addr != nil {
 			r.IPs = append(r.IPs, &current.IPConfig{
-				Version:   "4",
 				Interface: current.Int(0),
 				Address:   *v4Addr,
 			})
@@ -122,7 +129,6 @@ func cmdAdd(args *skel.CmdArgs) error {
 
 		if v6Addr != nil {
 			r.IPs = append(r.IPs, &current.IPConfig{
-				Version:   "6",
 				Interface: current.Int(0),
 				Address:   *v6Addr,
 			})
@@ -153,7 +159,14 @@ func cmdDel(args *skel.CmdArgs) error {
 		return nil
 	})
 	if err != nil {
-		return err // not tested
+		//  if NetNs is passed down by the Cloud Orchestration Engine, or if it called multiple times
+		// so don't return an error if the device is already removed.
+		// https://github.com/kubernetes/kubernetes/issues/43014#issuecomment-287164444
+		_, ok := err.(ns.NSPathNotExistErr)
+		if ok {
+			return nil
+		}
+		return err
 	}
 
 	return nil

plugins/main/loopback/loopback_test.go

@@ -28,6 +28,10 @@ import (
 	"github.com/onsi/gomega/gexec"
 )
 
+func generateConfig(cniVersion string) *strings.Reader {
+	return strings.NewReader(fmt.Sprintf(`{ "name": "loopback-test", "cniVersion": "%s" }`, cniVersion))
+}
+
 var _ = Describe("Loopback", func() {
 	var (
 		networkNS ns.NetNS
@@ -49,7 +53,6 @@ var _ = Describe("Loopback", func() {
 			fmt.Sprintf("CNI_ARGS=%s", "none"),
 			fmt.Sprintf("CNI_PATH=%s", "/some/test/path"),
 		}
-		command.Stdin = strings.NewReader(`{ "name": "loopback-test", "cniVersion": "0.1.0" }`)
 	})
 
 	AfterEach(func() {
@@ -57,45 +60,53 @@ var _ = Describe("Loopback", func() {
 		Expect(testutils.UnmountNS(networkNS)).To(Succeed())
 	})
 
-	Context("when given a network namespace", func() {
-		It("sets the lo device to UP", func() {
-			command.Env = append(environ, fmt.Sprintf("CNI_COMMAND=%s", "ADD"))
+	for _, ver := range testutils.AllSpecVersions {
+		// Redefine ver inside for scope so real value is picked up by each dynamically defined It()
+		// See Gingkgo's "Patterns for dynamically generating tests" documentation.
+		ver := ver
 
-			session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter)
-			Expect(err).NotTo(HaveOccurred())
+		Context("when given a network namespace", func() {
+			It(fmt.Sprintf("[%s] sets the lo device to UP", ver), func() {
+				command.Stdin = generateConfig(ver)
+				command.Env = append(environ, fmt.Sprintf("CNI_COMMAND=%s", "ADD"))
 
-			Eventually(session).Should(gbytes.Say(`{.*}`))
-			Eventually(session).Should(gexec.Exit(0))
+				session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter)
+				Expect(err).NotTo(HaveOccurred())
 
-			var lo *net.Interface
-			err = networkNS.Do(func(ns.NetNS) error {
-				var err error
-				lo, err = net.InterfaceByName("lo")
-				return err
+				Eventually(session).Should(gbytes.Say(`{.*}`))
+				Eventually(session).Should(gexec.Exit(0))
+
+				var lo *net.Interface
+				err = networkNS.Do(func(ns.NetNS) error {
+					var err error
+					lo, err = net.InterfaceByName("lo")
+					return err
+				})
+				Expect(err).NotTo(HaveOccurred())
+
+				Expect(lo.Flags & net.FlagUp).To(Equal(net.FlagUp))
 			})
-			Expect(err).NotTo(HaveOccurred())
 
-			Expect(lo.Flags & net.FlagUp).To(Equal(net.FlagUp))
-		})
+			It(fmt.Sprintf("[%s] sets the lo device to DOWN", ver), func() {
+				command.Stdin = generateConfig(ver)
+				command.Env = append(environ, fmt.Sprintf("CNI_COMMAND=%s", "DEL"))
 
-		It("sets the lo device to DOWN", func() {
-			command.Env = append(environ, fmt.Sprintf("CNI_COMMAND=%s", "DEL"))
+				session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter)
+				Expect(err).NotTo(HaveOccurred())
 
-			session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter)
-			Expect(err).NotTo(HaveOccurred())
+				Eventually(session).Should(gbytes.Say(``))
+				Eventually(session).Should(gexec.Exit(0))
 
-			Eventually(session).Should(gbytes.Say(``))
-			Eventually(session).Should(gexec.Exit(0))
+				var lo *net.Interface
+				err = networkNS.Do(func(ns.NetNS) error {
+					var err error
+					lo, err = net.InterfaceByName("lo")
+					return err
+				})
+				Expect(err).NotTo(HaveOccurred())
 
-			var lo *net.Interface
-			err = networkNS.Do(func(ns.NetNS) error {
-				var err error
-				lo, err = net.InterfaceByName("lo")
-				return err
+				Expect(lo.Flags & net.FlagUp).NotTo(Equal(net.FlagUp))
 			})
-			Expect(err).NotTo(HaveOccurred())
-
-			Expect(lo.Flags & net.FlagUp).NotTo(Equal(net.FlagUp))
 		})
-	})
+	}
 })

plugins/main/macvlan/README.md

@@ -1,34 +1,5 @@
-# macvlan plugin
 
-## Overview
+This document has moved to the [containernetworking/cni.dev](https://github.com/containernetworking/cni.dev) repo.
 
-[macvlan](http://backreference.org/2014/03/20/some-notes-on-macvlanmacvtap/) functions like a switch that is already connected to the host interface.
-A host interface gets "enslaved" with the virtual interfaces sharing the physical device but having distinct MAC addresses.
-Since each macvlan interface has its own MAC address, it makes it easy to use with existing DHCP servers already present on the network.
+You can find it online here: https://cni.dev/plugins/current/main/macvlan/
 
-## Example configuration
-
-```
-{
-	"name": "mynet",
-	"type": "macvlan",
-	"master": "eth0",
-	"ipam": {
-		"type": "dhcp"
-	}
-}
-```
-
-## Network configuration reference
-
-* `name` (string, required): the name of the network
-* `type` (string, required): "macvlan"
-* `master` (string, optional): name of the host interface to enslave. Defaults to default route interface.
-* `mode` (string, optional): one of "bridge", "private", "vepa", "passthru". Defaults to "bridge".
-* `mtu` (integer, optional): explicitly set MTU to the specified value. Defaults to the value chosen by the kernel. The value must be \[0, master's MTU\].
-* `ipam` (dictionary, required): IPAM configuration to be used for this network. For interface only without ip address, create empty dictionary.
-
-## Notes
-
-* If you are testing on a laptop, please remember that most wireless cards do not support being enslaved by macvlan.
-* A single master interface can not be enslaved by both `macvlan` and `ipvlan`.

plugins/main/macvlan/macvlan.go

@@ -21,12 +21,11 @@ import (
 	"net"
 	"runtime"
 
-	"github.com/j-keck/arping"
 	"github.com/vishvananda/netlink"
 
 	"github.com/containernetworking/cni/pkg/skel"
 	"github.com/containernetworking/cni/pkg/types"
-	"github.com/containernetworking/cni/pkg/types/current"
+	current "github.com/containernetworking/cni/pkg/types/100"
 	"github.com/containernetworking/cni/pkg/version"
 
 	"github.com/containernetworking/plugins/pkg/ip"
@@ -36,10 +35,6 @@ import (
 	"github.com/containernetworking/plugins/pkg/utils/sysctl"
 )
 
-const (
-	IPv4InterfaceArpProxySysctlTemplate = "net.ipv4.conf.%s.proxy_arp"
-)
-
 type NetConf struct {
 	types.NetConf
 	Master string `json:"master"`
@@ -208,14 +203,6 @@ func createMacvlan(conf *NetConf, ifName string, netns ns.NetNS) (*current.Inter
 	}
 
 	err = netns.Do(func(_ ns.NetNS) error {
-		// TODO: duplicate following lines for ipv6 support, when it will be added in other places
-		ipv4SysctlValueName := fmt.Sprintf(IPv4InterfaceArpProxySysctlTemplate, tmpName)
-		if _, err := sysctl.Sysctl(ipv4SysctlValueName, "1"); err != nil {
-			// remove the newly added link and ignore errors, because we already are in a failed state
-			_ = netlink.LinkDel(mv)
-			return fmt.Errorf("failed to set proxy_arp on newly added interface %q: %v", tmpName, err)
-		}
-
 		err := ip.RenameLink(tmpName, ifName)
 		if err != nil {
 			_ = netlink.LinkDel(mv)
@@ -269,7 +256,10 @@ func cmdAdd(args *skel.CmdArgs) error {
 	}()
 
 	// Assume L2 interface only
-	result := &current.Result{CNIVersion: cniVersion, Interfaces: []*current.Interface{macvlanInterface}}
+	result := &current.Result{
+		CNIVersion: current.ImplementedSpecVersion,
+		Interfaces: []*current.Interface{macvlanInterface},
+	}
 
 	if isLayer3 {
 		// run the IPAM plugin and get back the config to apply
@@ -304,20 +294,11 @@ func cmdAdd(args *skel.CmdArgs) error {
 		}
 
 		err = netns.Do(func(_ ns.NetNS) error {
+			_, _ = sysctl.Sysctl(fmt.Sprintf("net/ipv4/conf/%s/arp_notify", args.IfName), "1")
+
 			if err := ipam.ConfigureIface(args.IfName, result); err != nil {
 				return err
 			}
-
-			contVeth, err := net.InterfaceByName(args.IfName)
-			if err != nil {
-				return fmt.Errorf("failed to look up %q: %v", args.IfName, err)
-			}
-
-			for _, ipc := range result.IPs {
-				if ipc.Version == "4" {
-					_ = arping.GratuitousArpOverIface(ipc.Address.IP, *contVeth)
-				}
-			}
 			return nil
 		})
 		if err != nil {
@@ -377,6 +358,17 @@ func cmdDel(args *skel.CmdArgs) error {
 		return nil
 	})
 
+	if err != nil {
+		//  if NetNs is passed down by the Cloud Orchestration Engine, or if it called multiple times
+		// so don't return an error if the device is already removed.
+		// https://github.com/kubernetes/kubernetes/issues/43014#issuecomment-287164444
+		_, ok := err.(ns.NSPathNotExistErr)
+		if ok {
+			return nil
+		}
+		return err
+	}
+
 	return err
 }
 

plugins/main/ptp/README.md

@@ -1,32 +1,5 @@
-# ptp plugin
 
-## Overview
-The ptp plugin creates a point-to-point link between a container and the host by using a veth device.
-One end of the veth pair is placed inside a container and the other end resides on the host.
-The host-local IPAM plugin can be used to allocate an IP address to the container.
-The traffic of the container interface will be routed through the interface of the host.
+This document has moved to the [containernetworking/cni.dev](https://github.com/containernetworking/cni.dev) repo.
 
-## Example network configuration
+You can find it online here: https://cni.dev/plugins/current/main/ptp/
 
-```
-{
-	"name": "mynet",
-	"type": "ptp",
-	"ipam": {
-		"type": "host-local",
-		"subnet": "10.1.1.0/24"
-	},
-	"dns": {
-		"nameservers": [ "10.1.1.1", "8.8.8.8" ]
-	}
-}
-```
-
-## Network configuration reference
-
-* `name` (string, required): the name of the network
-* `type` (string, required): "ptp"
-* `ipMasq` (boolean, optional): set up IP Masquerade on the host for traffic originating from ip of this network and destined outside of this network. Defaults to false.
-* `mtu` (integer, optional): explicitly set MTU to the specified value. Defaults to value chosen by the kernel.
-* `ipam` (dictionary, required): IPAM configuration to be used for this network.
-* `dns` (dictionary, optional): DNS information to return as described in the [Result](https://github.com/containernetworking/cni/blob/master/SPEC.md#result).

plugins/main/ptp/ptp.go

@@ -22,12 +22,11 @@ import (
 	"os"
 	"runtime"
 
-	"github.com/j-keck/arping"
 	"github.com/vishvananda/netlink"
 
 	"github.com/containernetworking/cni/pkg/skel"
 	"github.com/containernetworking/cni/pkg/types"
-	"github.com/containernetworking/cni/pkg/types/current"
+	current "github.com/containernetworking/cni/pkg/types/100"
 	"github.com/containernetworking/cni/pkg/version"
 
 	"github.com/containernetworking/plugins/pkg/ip"
@@ -66,7 +65,7 @@ func setupContainerVeth(netns ns.NetNS, ifName string, mtu int, pr *current.Resu
 	containerInterface := &current.Interface{}
 
 	err := netns.Do(func(hostNS ns.NetNS) error {
-		hostVeth, contVeth0, err := ip.SetupVeth(ifName, mtu, hostNS)
+		hostVeth, contVeth0, err := ip.SetupVeth(ifName, mtu, "", hostNS)
 		if err != nil {
 			return err
 		}
@@ -83,15 +82,15 @@ func setupContainerVeth(netns ns.NetNS, ifName string, mtu int, pr *current.Resu
 
 		pr.Interfaces = []*current.Interface{hostInterface, containerInterface}
 
-		if err = ipam.ConfigureIface(ifName, pr); err != nil {
-			return err
-		}
-
 		contVeth, err := net.InterfaceByName(ifName)
 		if err != nil {
 			return fmt.Errorf("failed to look up %q: %v", ifName, err)
 		}
 
+		if err = ipam.ConfigureIface(ifName, pr); err != nil {
+			return err
+		}
+
 		for _, ipc := range pr.IPs {
 			// Delete the route that was automatically added
 			route := netlink.Route{
@@ -108,7 +107,7 @@ func setupContainerVeth(netns ns.NetNS, ifName string, mtu int, pr *current.Resu
 			}
 
 			addrBits := 32
-			if ipc.Version == "6" {
+			if ipc.Address.IP.To4() == nil {
 				addrBits = 128
 			}
 
@@ -139,13 +138,6 @@ func setupContainerVeth(netns ns.NetNS, ifName string, mtu int, pr *current.Resu
 			}
 		}
 
-		// Send a gratuitous arp for all v4 addresses
-		for _, ipc := range pr.IPs {
-			if ipc.Version == "4" {
-				_ = arping.GratuitousArpOverIface(ipc.Address.IP, *contVeth)
-			}
-		}
-
 		return nil
 	})
 	if err != nil {
@@ -292,6 +284,13 @@ func cmdDel(args *skel.CmdArgs) error {
 	})
 
 	if err != nil {
+		//  if NetNs is passed down by the Cloud Orchestration Engine, or if it called multiple times
+		// so don't return an error if the device is already removed.
+		// https://github.com/kubernetes/kubernetes/issues/43014#issuecomment-287164444
+		_, ok := err.(ns.NSPathNotExistErr)
+		if ok {
+			return nil
+		}
 		return err
 	}
 

plugins/main/ptp/ptp_test.go

@@ -17,11 +17,15 @@ package main
 import (
 	"encoding/json"
 	"fmt"
+	"io/ioutil"
 	"os"
+	"strings"
 
 	"github.com/containernetworking/cni/pkg/skel"
 	"github.com/containernetworking/cni/pkg/types"
-	"github.com/containernetworking/cni/pkg/types/current"
+	"github.com/containernetworking/cni/pkg/types/020"
+	"github.com/containernetworking/cni/pkg/types/040"
+	"github.com/containernetworking/cni/pkg/types/100"
 	"github.com/containernetworking/plugins/pkg/ns"
 	"github.com/containernetworking/plugins/pkg/testutils"
 
@@ -41,7 +45,7 @@ type Net struct {
 	IPAM          *allocator.IPAMConfig  `json:"ipam"`
 	DNS           types.DNS              `json:"dns"`
 	RawPrevResult map[string]interface{} `json:"prevResult,omitempty"`
-	PrevResult    current.Result         `json:"-"`
+	PrevResult    types100.Result        `json:"-"`
 }
 
 func buildOneConfig(netName string, cniVersion string, orig *Net, prevResult types.Result) (*Net, error) {
@@ -87,43 +91,166 @@ func buildOneConfig(netName string, cniVersion string, orig *Net, prevResult typ
 
 }
 
+type tester interface {
+	// verifyResult minimally verifies the Result and returns the interface's IP addresses and MAC address
+	verifyResult(result types.Result, expectedIfName, expectedSandbox string, expectedDNS types.DNS) ([]resultIP, string)
+}
+
+type testerBase struct{}
+
+type testerV10x testerBase
+type testerV04x testerBase
+type testerV03x testerBase
+type testerV01xOr02x testerBase
+
+func newTesterByVersion(version string) tester {
+	switch {
+	case strings.HasPrefix(version, "1.0."):
+		return &testerV10x{}
+	case strings.HasPrefix(version, "0.4."):
+		return &testerV04x{}
+	case strings.HasPrefix(version, "0.3."):
+		return &testerV03x{}
+	default:
+		return &testerV01xOr02x{}
+	}
+}
+
+type resultIP struct {
+	ip string
+	gw string
+}
+
+// verifyResult minimally verifies the Result and returns the interface's IP addresses and MAC address
+func (t *testerV10x) verifyResult(result types.Result, expectedIfName, expectedSandbox string, expectedDNS types.DNS) ([]resultIP, string) {
+	r, err := types100.GetResult(result)
+	Expect(err).NotTo(HaveOccurred())
+
+	Expect(r.Interfaces).To(HaveLen(2))
+	Expect(r.Interfaces[0].Name).To(HavePrefix("veth"))
+	Expect(r.Interfaces[0].Mac).To(HaveLen(17))
+	Expect(r.Interfaces[0].Sandbox).To(BeEmpty())
+	Expect(r.Interfaces[1].Name).To(Equal(expectedIfName))
+	Expect(r.Interfaces[1].Sandbox).To(Equal(expectedSandbox))
+
+	Expect(r.DNS).To(Equal(expectedDNS))
+
+	// Grab IPs from container interface
+	ips := []resultIP{}
+	for _, ipc := range r.IPs {
+		if *ipc.Interface == 1 {
+			ips = append(ips, resultIP{
+				ip: ipc.Address.IP.String(),
+				gw: ipc.Gateway.String(),
+			})
+		}
+	}
+
+	return ips, r.Interfaces[1].Mac
+}
+
+func verify0403(result types.Result, expectedIfName, expectedSandbox string, expectedDNS types.DNS) ([]resultIP, string) {
+	r, err := types040.GetResult(result)
+	Expect(err).NotTo(HaveOccurred())
+
+	Expect(r.Interfaces).To(HaveLen(2))
+	Expect(r.Interfaces[0].Name).To(HavePrefix("veth"))
+	Expect(r.Interfaces[0].Mac).To(HaveLen(17))
+	Expect(r.Interfaces[0].Sandbox).To(BeEmpty())
+	Expect(r.Interfaces[1].Name).To(Equal(expectedIfName))
+	Expect(r.Interfaces[1].Sandbox).To(Equal(expectedSandbox))
+
+	Expect(r.DNS).To(Equal(expectedDNS))
+
+	// Grab IPs from container interface
+	ips := []resultIP{}
+	for _, ipc := range r.IPs {
+		if *ipc.Interface == 1 {
+			ips = append(ips, resultIP{
+				ip: ipc.Address.IP.String(),
+				gw: ipc.Gateway.String(),
+			})
+		}
+	}
+
+	return ips, r.Interfaces[1].Mac
+}
+
+// verifyResult minimally verifies the Result and returns the interface's IP addresses and MAC address
+func (t *testerV04x) verifyResult(result types.Result, expectedIfName, expectedSandbox string, expectedDNS types.DNS) ([]resultIP, string) {
+	return verify0403(result, expectedIfName, expectedSandbox, expectedDNS)
+}
+
+// verifyResult minimally verifies the Result and returns the interface's IP addresses and MAC address
+func (t *testerV03x) verifyResult(result types.Result, expectedIfName, expectedSandbox string, expectedDNS types.DNS) ([]resultIP, string) {
+	return verify0403(result, expectedIfName, expectedSandbox, expectedDNS)
+}
+
+// verifyResult minimally verifies the Result and returns the interface's IP addresses and MAC address
+func (t *testerV01xOr02x) verifyResult(result types.Result, expectedIfName, expectedSandbox string, expectedDNS types.DNS) ([]resultIP, string) {
+	r, err := types020.GetResult(result)
+	Expect(err).NotTo(HaveOccurred())
+
+	ips := []resultIP{}
+	if r.IP4 != nil && r.IP4.IP.IP != nil {
+		ips = append(ips, resultIP{
+			ip: r.IP4.IP.IP.String(),
+			gw: r.IP4.Gateway.String(),
+		})
+	}
+	if r.IP6 != nil && r.IP6.IP.IP != nil {
+		ips = append(ips, resultIP{
+			ip: r.IP6.IP.IP.String(),
+			gw: r.IP6.Gateway.String(),
+		})
+	}
+
+	// 0.2 and earlier don't return MAC address
+	return ips, ""
+}
+
 var _ = Describe("ptp Operations", func() {
-	var originalNS ns.NetNS
+	var originalNS, targetNS ns.NetNS
+	var dataDir string
 
 	BeforeEach(func() {
 		// Create a new NetNS so we don't modify the host
 		var err error
 		originalNS, err = testutils.NewNS()
 		Expect(err).NotTo(HaveOccurred())
+		targetNS, err = testutils.NewNS()
+		Expect(err).NotTo(HaveOccurred())
+
+		dataDir, err = ioutil.TempDir("", "ptp_test")
+		Expect(err).NotTo(HaveOccurred())
 	})
 
 	AfterEach(func() {
+		Expect(os.RemoveAll(dataDir)).To(Succeed())
 		Expect(originalNS.Close()).To(Succeed())
 		Expect(testutils.UnmountNS(originalNS)).To(Succeed())
+		Expect(targetNS.Close()).To(Succeed())
+		Expect(testutils.UnmountNS(targetNS)).To(Succeed())
 	})
 
-	doTest := func(conf string, numIPs int, expectedDNSConf types.DNS) {
+	doTest := func(conf, cniVersion string, numIPs int, expectedDNSConf types.DNS, targetNS ns.NetNS) {
 		const IFNAME = "ptp0"
 
-		targetNs, err := testutils.NewNS()
-		Expect(err).NotTo(HaveOccurred())
-		defer targetNs.Close()
-
 		args := &skel.CmdArgs{
 			ContainerID: "dummy",
-			Netns:       targetNs.Path(),
+			Netns:       targetNS.Path(),
 			IfName:      IFNAME,
 			StdinData:   []byte(conf),
 		}
 
-		var resI types.Result
-		var res *current.Result
+		var result types.Result
 
 		// Execute the plugin with the ADD command, creating the veth endpoints
-		err = originalNS.Do(func(ns.NetNS) error {
+		err := originalNS.Do(func(ns.NetNS) error {
 			defer GinkgoRecover()
 
-			resI, _, err = testutils.CmdAddWithArgs(args, func() error {
+			var err error
+			result, _, err = testutils.CmdAddWithArgs(args, func() error {
 				return cmdAdd(args)
 			})
 			Expect(err).NotTo(HaveOccurred())
@@ -131,32 +258,25 @@ var _ = Describe("ptp Operations", func() {
 		})
 		Expect(err).NotTo(HaveOccurred())
 
-		res, err = current.NewResultFromResult(resI)
-		Expect(err).NotTo(HaveOccurred())
+		t := newTesterByVersion(cniVersion)
+		ips, mac := t.verifyResult(result, IFNAME, targetNS.Path(), expectedDNSConf)
+		Expect(len(ips)).To(Equal(numIPs))
 
 		// Make sure ptp link exists in the target namespace
 		// Then, ping the gateway
-		seenIPs := 0
-
-		wantMac := ""
-		err = targetNs.Do(func(ns.NetNS) error {
+		err = targetNS.Do(func(ns.NetNS) error {
 			defer GinkgoRecover()
 
 			link, err := netlink.LinkByName(IFNAME)
 			Expect(err).NotTo(HaveOccurred())
-			wantMac = link.Attrs().HardwareAddr.String()
+			if mac != "" {
+				Expect(mac).To(Equal(link.Attrs().HardwareAddr.String()))
+			}
 
-			for _, ipc := range res.IPs {
-				if *ipc.Interface != 1 {
-					continue
-				}
-				seenIPs += 1
-				saddr := ipc.Address.IP.String()
-				daddr := ipc.Gateway.String()
-				fmt.Fprintln(GinkgoWriter, "ping", saddr, "->", daddr)
-
-				if err := testutils.Ping(saddr, daddr, (ipc.Version == "6"), 30); err != nil {
-					return fmt.Errorf("ping %s -> %s failed: %s", saddr, daddr, err)
+			for _, ipc := range ips {
+				fmt.Fprintln(GinkgoWriter, "ping", ipc.ip, "->", ipc.gw)
+				if err := testutils.Ping(ipc.ip, ipc.gw, 30); err != nil {
+					return fmt.Errorf("ping %s -> %s failed: %s", ipc.ip, ipc.gw, err)
 				}
 			}
 
@@ -164,121 +284,6 @@ var _ = Describe("ptp Operations", func() {
 		})
 		Expect(err).NotTo(HaveOccurred())
 
-		Expect(seenIPs).To(Equal(numIPs))
-
-		// make sure the interfaces are correct
-		Expect(res.Interfaces).To(HaveLen(2))
-
-		Expect(res.Interfaces[0].Name).To(HavePrefix("veth"))
-		Expect(res.Interfaces[0].Mac).To(HaveLen(17))
-		Expect(res.Interfaces[0].Sandbox).To(BeEmpty())
-
-		Expect(res.Interfaces[1].Name).To(Equal(IFNAME))
-		Expect(res.Interfaces[1].Mac).To(Equal(wantMac))
-		Expect(res.Interfaces[1].Sandbox).To(Equal(targetNs.Path()))
-
-		// make sure DNS is correct
-		Expect(res.DNS).To(Equal(expectedDNSConf))
-
-		// Call the plugins with the DEL command, deleting the veth endpoints
-		err = originalNS.Do(func(ns.NetNS) error {
-			defer GinkgoRecover()
-
-			err := testutils.CmdDelWithArgs(args, func() error {
-				return cmdDel(args)
-			})
-			Expect(err).NotTo(HaveOccurred())
-			return nil
-		})
-		Expect(err).NotTo(HaveOccurred())
-
-		// Make sure ptp link has been deleted
-		err = targetNs.Do(func(ns.NetNS) error {
-			defer GinkgoRecover()
-
-			link, err := netlink.LinkByName(IFNAME)
-			Expect(err).To(HaveOccurred())
-			Expect(link).To(BeNil())
-			return nil
-		})
-		Expect(err).NotTo(HaveOccurred())
-	}
-
-	doTestv4 := func(conf string, netName string, numIPs int) {
-		const IFNAME = "ptp0"
-
-		targetNs, err := testutils.NewNS()
-		Expect(err).NotTo(HaveOccurred())
-		defer targetNs.Close()
-
-		args := &skel.CmdArgs{
-			ContainerID: "dummy",
-			Netns:       targetNs.Path(),
-			IfName:      IFNAME,
-			StdinData:   []byte(conf),
-		}
-
-		var resI types.Result
-		var res *current.Result
-
-		// Execute the plugin with the ADD command, creating the veth endpoints
-		err = originalNS.Do(func(ns.NetNS) error {
-			defer GinkgoRecover()
-
-			resI, _, err = testutils.CmdAddWithArgs(args, func() error {
-				return cmdAdd(args)
-			})
-			Expect(err).NotTo(HaveOccurred())
-			return nil
-		})
-		Expect(err).NotTo(HaveOccurred())
-
-		res, err = current.NewResultFromResult(resI)
-		Expect(err).NotTo(HaveOccurred())
-
-		// Make sure ptp link exists in the target namespace
-		// Then, ping the gateway
-		seenIPs := 0
-
-		wantMac := ""
-		err = targetNs.Do(func(ns.NetNS) error {
-			defer GinkgoRecover()
-
-			link, err := netlink.LinkByName(IFNAME)
-			Expect(err).NotTo(HaveOccurred())
-			wantMac = link.Attrs().HardwareAddr.String()
-
-			for _, ipc := range res.IPs {
-				if *ipc.Interface != 1 {
-					continue
-				}
-				seenIPs += 1
-				saddr := ipc.Address.IP.String()
-				daddr := ipc.Gateway.String()
-				fmt.Fprintln(GinkgoWriter, "ping", saddr, "->", daddr)
-
-				if err := testutils.Ping(saddr, daddr, (ipc.Version == "6"), 30); err != nil {
-					return fmt.Errorf("ping %s -> %s failed: %s", saddr, daddr, err)
-				}
-			}
-
-			return nil
-		})
-		Expect(err).NotTo(HaveOccurred())
-
-		Expect(seenIPs).To(Equal(numIPs))
-
-		// make sure the interfaces are correct
-		Expect(res.Interfaces).To(HaveLen(2))
-
-		Expect(res.Interfaces[0].Name).To(HavePrefix("veth"))
-		Expect(res.Interfaces[0].Mac).To(HaveLen(17))
-		Expect(res.Interfaces[0].Sandbox).To(BeEmpty())
-
-		Expect(res.Interfaces[1].Name).To(Equal(IFNAME))
-		Expect(res.Interfaces[1].Mac).To(Equal(wantMac))
-		Expect(res.Interfaces[1].Sandbox).To(Equal(targetNs.Path()))
-
 		// call CmdCheck
 		n := &Net{}
 		err = json.Unmarshal([]byte(conf), &n)
@@ -287,8 +292,7 @@ var _ = Describe("ptp Operations", func() {
 		n.IPAM, _, err = allocator.LoadIPAMConfig([]byte(conf), "")
 		Expect(err).NotTo(HaveOccurred())
 
-		cniVersion := "0.4.0"
-		newConf, err := buildOneConfig(netName, cniVersion, n, res)
+		newConf, err := buildOneConfig(n.Name, cniVersion, n, result)
 		Expect(err).NotTo(HaveOccurred())
 
 		confString, err := json.Marshal(newConf)
@@ -299,11 +303,13 @@ var _ = Describe("ptp Operations", func() {
 		// CNI Check host-device in the target namespace
 		err = originalNS.Do(func(ns.NetNS) error {
 			defer GinkgoRecover()
-			var err error
-			err = testutils.CmdCheckWithArgs(args, func() error { return cmdCheck(args) })
-			return err
+			return testutils.CmdCheckWithArgs(args, func() error { return cmdCheck(args) })
 		})
-		Expect(err).NotTo(HaveOccurred())
+		if testutils.SpecVersionHasCHECK(cniVersion) {
+			Expect(err).NotTo(HaveOccurred())
+		} else {
+			Expect(err).To(MatchError("config version does not allow CHECK"))
+		}
 
 		args.StdinData = []byte(conf)
 
@@ -320,7 +326,7 @@ var _ = Describe("ptp Operations", func() {
 		Expect(err).NotTo(HaveOccurred())
 
 		// Make sure ptp link has been deleted
-		err = targetNs.Do(func(ns.NetNS) error {
+		err = targetNS.Do(func(ns.NetNS) error {
 			defer GinkgoRecover()
 
 			link, err := netlink.LinkByName(IFNAME)
@@ -331,227 +337,200 @@ var _ = Describe("ptp Operations", func() {
 		Expect(err).NotTo(HaveOccurred())
 	}
 
-	It("configures and deconfigures a ptp link with ADD/DEL", func() {
-		dnsConf := types.DNS{
-			Nameservers: []string{"10.1.2.123"},
-			Domain:      "some.domain.test",
-			Search:      []string{"search.test"},
-			Options:     []string{"option1:foo"},
-		}
-		dnsConfBytes, err := json.Marshal(dnsConf)
-		Expect(err).NotTo(HaveOccurred())
+	for _, ver := range testutils.AllSpecVersions {
+		// Redefine ver inside for scope so real value is picked up by each dynamically defined It()
+		// See Gingkgo's "Patterns for dynamically generating tests" documentation.
+		ver := ver
 
-		conf := fmt.Sprintf(`{
-    "cniVersion": "0.3.1",
-    "name": "mynet",
-    "type": "ptp",
-    "ipMasq": true,
-    "mtu": 5000,
-    "ipam": {
-        "type": "host-local",
-        "subnet": "10.1.2.0/24"
-    },
-    "dns": %s
-}`, string(dnsConfBytes))
-
-		doTest(conf, 1, dnsConf)
-	})
-
-	It("configures and deconfigures a dual-stack ptp link with ADD/DEL", func() {
-		conf := `{
-    "cniVersion": "0.3.1",
-    "name": "mynet",
-    "type": "ptp",
-    "ipMasq": true,
-    "mtu": 5000,
-    "ipam": {
-        "type": "host-local",
-		"ranges": [
-			[{ "subnet": "10.1.2.0/24"}],
-			[{ "subnet": "2001:db8:1::0/66"}]
-		]
-    }
-}`
-
-		doTest(conf, 2, types.DNS{})
-	})
-
-	It("does not override IPAM DNS settings if no DNS settings provided", func() {
-		ipamDNSConf := types.DNS{
-			Nameservers: []string{"10.1.2.123"},
-			Domain:      "some.domain.test",
-			Search:      []string{"search.test"},
-			Options:     []string{"option1:foo"},
-		}
-		resolvConfPath, err := testutils.TmpResolvConf(ipamDNSConf)
-		Expect(err).NotTo(HaveOccurred())
-		defer os.RemoveAll(resolvConfPath)
-
-		conf := fmt.Sprintf(`{
-    "cniVersion": "0.3.1",
-    "name": "mynet",
-    "type": "ptp",
-    "ipMasq": true,
-    "mtu": 5000,
-    "ipam": {
-        "type": "host-local",
-        "subnet": "10.1.2.0/24",
-        "resolvConf": "%s"
-    }
-}`, resolvConfPath)
-
-		doTest(conf, 1, ipamDNSConf)
-	})
-
-	It("overrides IPAM DNS settings if any DNS settings provided", func() {
-		ipamDNSConf := types.DNS{
-			Nameservers: []string{"10.1.2.123"},
-			Domain:      "some.domain.test",
-			Search:      []string{"search.test"},
-			Options:     []string{"option1:foo"},
-		}
-		resolvConfPath, err := testutils.TmpResolvConf(ipamDNSConf)
-		Expect(err).NotTo(HaveOccurred())
-		defer os.RemoveAll(resolvConfPath)
-
-		for _, ptpDNSConf := range []types.DNS{
-			{
-				Nameservers: []string{"10.1.2.234"},
-			},
-			{
-				Domain: "someother.domain.test",
-			},
-			{
-				Search: []string{"search.elsewhere.test"},
-			},
-			{
-				Options: []string{"option2:bar"},
-			},
-		} {
-			dnsConfBytes, err := json.Marshal(ptpDNSConf)
+		It(fmt.Sprintf("[%s] configures and deconfigures a ptp link with ADD/DEL", ver), func() {
+			dnsConf := types.DNS{
+				Nameservers: []string{"10.1.2.123"},
+				Domain:      "some.domain.test",
+				Search:      []string{"search.test"},
+				Options:     []string{"option1:foo"},
+			}
+			dnsConfBytes, err := json.Marshal(dnsConf)
 			Expect(err).NotTo(HaveOccurred())
 
 			conf := fmt.Sprintf(`{
-    "cniVersion": "0.3.1",
-    "name": "mynet",
-    "type": "ptp",
-    "ipMasq": true,
-    "mtu": 5000,
-    "ipam": {
-        "type": "host-local",
-        "subnet": "10.1.2.0/24",
-        "resolvConf": "%s"
-    },
-    "dns": %s
-}`, resolvConfPath, string(dnsConfBytes))
+			    "cniVersion": "%s",
+			    "name": "mynet",
+			    "type": "ptp",
+			    "ipMasq": true,
+			    "mtu": 5000,
+			    "ipam": {
+				"type": "host-local",
+				"subnet": "10.1.2.0/24",
+				"dataDir": "%s"
+			    },
+			    "dns": %s
+			}`, ver, dataDir, string(dnsConfBytes))
 
-			doTest(conf, 1, ptpDNSConf)
-		}
-	})
+			doTest(conf, ver, 1, dnsConf, targetNS)
+		})
 
-	It("overrides IPAM DNS settings if any empty list DNS settings provided", func() {
-		ipamDNSConf := types.DNS{
-			Nameservers: []string{"10.1.2.123"},
-			Domain:      "some.domain.test",
-			Search:      []string{"search.test"},
-			Options:     []string{"option1:foo"},
-		}
-		resolvConfPath, err := testutils.TmpResolvConf(ipamDNSConf)
-		Expect(err).NotTo(HaveOccurred())
-		defer os.RemoveAll(resolvConfPath)
+		It(fmt.Sprintf("[%s] configures and deconfigures a dual-stack ptp link with ADD/DEL", ver), func() {
+			conf := fmt.Sprintf(`{
+			    "cniVersion": "%s",
+			    "name": "mynet",
+			    "type": "ptp",
+			    "ipMasq": true,
+			    "mtu": 5000,
+			    "ipam": {
+				"type": "host-local",
+				"ranges": [
+					[{ "subnet": "10.1.2.0/24"}],
+					[{ "subnet": "2001:db8:1::0/66"}]
+				],
+				"dataDir": "%s"
+			    }
+			}`, ver, dataDir)
 
-		conf := fmt.Sprintf(`{
-    "cniVersion": "0.3.1",
-    "name": "mynet",
-    "type": "ptp",
-    "ipMasq": true,
-    "mtu": 5000,
-    "ipam": {
-        "type": "host-local",
-        "subnet": "10.1.2.0/24",
-        "resolvConf": "%s"
-    },
-    "dns": {
-        "nameservers": [],
-        "search": [],
-        "options": []
-    }
-}`, resolvConfPath)
+			doTest(conf, ver, 2, types.DNS{}, targetNS)
+		})
 
-		doTest(conf, 1, types.DNS{})
-	})
+		It(fmt.Sprintf("[%s] does not override IPAM DNS settings if no DNS settings provided", ver), func() {
+			ipamDNSConf := types.DNS{
+				Nameservers: []string{"10.1.2.123"},
+				Domain:      "some.domain.test",
+				Search:      []string{"search.test"},
+				Options:     []string{"option1:foo"},
+			}
+			resolvConfPath, err := testutils.TmpResolvConf(ipamDNSConf)
+			Expect(err).NotTo(HaveOccurred())
+			defer os.RemoveAll(resolvConfPath)
 
-	It("deconfigures an unconfigured ptp link with DEL", func() {
-		const IFNAME = "ptp0"
+			conf := fmt.Sprintf(`{
+			    "cniVersion": "%s",
+			    "name": "mynet",
+			    "type": "ptp",
+			    "ipMasq": true,
+			    "mtu": 5000,
+			    "ipam": {
+				"type": "host-local",
+				"subnet": "10.1.2.0/24",
+				"resolvConf": "%s",
+				"dataDir": "%s"
+			    }
+			}`, ver, resolvConfPath, dataDir)
 
-		conf := `{
-    "cniVersion": "0.3.0",
-    "name": "mynet",
-    "type": "ptp",
-    "ipMasq": true,
-    "mtu": 5000,
-    "ipam": {
-        "type": "host-local",
-        "subnet": "10.1.2.0/24"
-    }
-}`
+			doTest(conf, ver, 1, ipamDNSConf, targetNS)
+		})
 
-		targetNs, err := testutils.NewNS()
-		Expect(err).NotTo(HaveOccurred())
-		defer targetNs.Close()
+		It(fmt.Sprintf("[%s] overrides IPAM DNS settings if any DNS settings provided", ver), func() {
+			ipamDNSConf := types.DNS{
+				Nameservers: []string{"10.1.2.123"},
+				Domain:      "some.domain.test",
+				Search:      []string{"search.test"},
+				Options:     []string{"option1:foo"},
+			}
+			resolvConfPath, err := testutils.TmpResolvConf(ipamDNSConf)
+			Expect(err).NotTo(HaveOccurred())
+			defer os.RemoveAll(resolvConfPath)
 
-		args := &skel.CmdArgs{
-			ContainerID: "dummy",
-			Netns:       targetNs.Path(),
-			IfName:      IFNAME,
-			StdinData:   []byte(conf),
-		}
+			for _, ptpDNSConf := range []types.DNS{
+				{
+					Nameservers: []string{"10.1.2.234"},
+				},
+				{
+					Domain: "someother.domain.test",
+				},
+				{
+					Search: []string{"search.elsewhere.test"},
+				},
+				{
+					Options: []string{"option2:bar"},
+				},
+			} {
+				dnsConfBytes, err := json.Marshal(ptpDNSConf)
+				Expect(err).NotTo(HaveOccurred())
 
-		// Call the plugins with the DEL command. It should not error even though the veth doesn't exist.
-		err = originalNS.Do(func(ns.NetNS) error {
-			defer GinkgoRecover()
+				conf := fmt.Sprintf(`{
+				    "cniVersion": "%s",
+				    "name": "mynet",
+				    "type": "ptp",
+				    "ipMasq": true,
+				    "mtu": 5000,
+				    "ipam": {
+					"type": "host-local",
+					"subnet": "10.1.2.0/24",
+					"resolvConf": "%s",
+					"dataDir": "%s"
+				    },
+				    "dns": %s
+				}`, ver, resolvConfPath, dataDir, string(dnsConfBytes))
 
-			err := testutils.CmdDelWithArgs(args, func() error {
-				return cmdDel(args)
+				doTest(conf, ver, 1, ptpDNSConf, targetNS)
+			}
+		})
+
+		It(fmt.Sprintf("[%s] overrides IPAM DNS settings if any empty list DNS settings provided", ver), func() {
+			ipamDNSConf := types.DNS{
+				Nameservers: []string{"10.1.2.123"},
+				Domain:      "some.domain.test",
+				Search:      []string{"search.test"},
+				Options:     []string{"option1:foo"},
+			}
+			resolvConfPath, err := testutils.TmpResolvConf(ipamDNSConf)
+			Expect(err).NotTo(HaveOccurred())
+			defer os.RemoveAll(resolvConfPath)
+
+			conf := fmt.Sprintf(`{
+			    "cniVersion": "%s",
+			    "name": "mynet",
+			    "type": "ptp",
+			    "ipMasq": true,
+			    "mtu": 5000,
+			    "ipam": {
+				"type": "host-local",
+				"subnet": "10.1.2.0/24",
+				"dataDir": "%s",
+				"resolvConf": "%s"
+			    },
+			    "dns": {
+				"nameservers": [],
+				"search": [],
+				"options": []
+			    }
+			}`, ver, dataDir, resolvConfPath)
+
+			doTest(conf, ver, 1, types.DNS{}, targetNS)
+		})
+
+		It(fmt.Sprintf("[%s] deconfigures an unconfigured ptp link with DEL", ver), func() {
+			const IFNAME = "ptp0"
+
+			conf := fmt.Sprintf(`{
+			    "cniVersion": "%s",
+			    "name": "mynet",
+			    "type": "ptp",
+			    "ipMasq": true,
+			    "mtu": 5000,
+			    "ipam": {
+				"type": "host-local",
+				"dataDir": "%s",
+				"subnet": "10.1.2.0/24"
+			    }
+			}`, ver, dataDir)
+
+			args := &skel.CmdArgs{
+				ContainerID: "dummy",
+				Netns:       targetNS.Path(),
+				IfName:      IFNAME,
+				StdinData:   []byte(conf),
+			}
+
+			// Call the plugins with the DEL command. It should not error even though the veth doesn't exist.
+			err := originalNS.Do(func(ns.NetNS) error {
+				defer GinkgoRecover()
+
+				err := testutils.CmdDelWithArgs(args, func() error {
+					return cmdDel(args)
+				})
+				Expect(err).NotTo(HaveOccurred())
+				return nil
 			})
 			Expect(err).NotTo(HaveOccurred())
-			return nil
 		})
-		Expect(err).NotTo(HaveOccurred())
-	})
-
-	It("configures and deconfigures a CNI V4 ptp link with ADD/DEL", func() {
-		conf := `{
-    "cniVersion": "0.4.0",
-    "name": "ptpNetv4",
-    "type": "ptp",
-    "ipMasq": true,
-    "mtu": 5000,
-    "ipam": {
-        "type": "host-local",
-        "subnet": "10.1.2.0/24"
-    }
-}`
-
-		doTestv4(conf, "ptpNetv4", 1)
-	})
-
-	It("configures and deconfigures a CNI V4 dual-stack ptp link with ADD/DEL", func() {
-		conf := `{
-    "cniVersion": "0.4.0",
-    "name": "ptpNetv4ds",
-    "type": "ptp",
-    "ipMasq": true,
-    "mtu": 5000,
-    "ipam": {
-        "type": "host-local",
-		"ranges": [
-			[{ "subnet": "10.1.2.0/24"}],
-			[{ "subnet": "2001:db8:1::0/66"}]
-		]
-    }
-}`
-
-		doTestv4(conf, "ptpNetv4ds", 2)
-	})
+	}
 })

plugins/main/vlan/vlan.go

@@ -24,7 +24,7 @@ import (
 
 	"github.com/containernetworking/cni/pkg/skel"
 	"github.com/containernetworking/cni/pkg/types"
-	"github.com/containernetworking/cni/pkg/types/current"
+	current "github.com/containernetworking/cni/pkg/types/100"
 	"github.com/containernetworking/cni/pkg/version"
 
 	"github.com/containernetworking/plugins/pkg/ip"
@@ -152,7 +152,7 @@ func cmdAdd(args *skel.CmdArgs) error {
 	// run the IPAM plugin and get back the config to apply
 	r, err := ipam.ExecAdd(n.IPAM.Type, args.StdinData)
 	if err != nil {
-		return err
+		return fmt.Errorf("failed to execute IPAM delegate: %v", err)
 	}
 
 	// Invoke ipam del if err to avoid ip leak
@@ -213,6 +213,17 @@ func cmdDel(args *skel.CmdArgs) error {
 		return err
 	})
 
+	if err != nil {
+		//  if NetNs is passed down by the Cloud Orchestration Engine, or if it called multiple times
+		// so don't return an error if the device is already removed.
+		// https://github.com/kubernetes/kubernetes/issues/43014#issuecomment-287164444
+		_, ok := err.(ns.NSPathNotExistErr)
+		if ok {
+			return nil
+		}
+		return err
+	}
+
 	return err
 }
 
@@ -238,7 +249,7 @@ func cmdCheck(args *skel.CmdArgs) error {
 		return err
 	}
 	if conf.NetConf.RawPrevResult == nil {
-		return fmt.Errorf("ptp: Required prevResult missing")
+		return fmt.Errorf("vlan: Required prevResult missing")
 	}
 	if err := version.ParsePrevResult(&conf.NetConf); err != nil {
 		return err
@@ -308,10 +319,10 @@ func validateCniContainerInterface(intf current.Interface, masterIndex int, vlan
 	}
 	link, err = netlink.LinkByName(intf.Name)
 	if err != nil {
-		return fmt.Errorf("ptp: Container Interface name in prevResult: %s not found", intf.Name)
+		return fmt.Errorf("vlan: Container Interface name in prevResult: %s not found", intf.Name)
 	}
 	if intf.Sandbox == "" {
-		return fmt.Errorf("ptp: Error: Container interface %s should not be in host namespace", link.Attrs().Name)
+		return fmt.Errorf("vlan: Error: Container interface %s should not be in host namespace", link.Attrs().Name)
 	}
 
 	vlan, isVlan := link.(*netlink.Vlan)

plugins/main/vlan/vlan_test.go

@@ -17,12 +17,17 @@ package main
 import (
 	"encoding/json"
 	"fmt"
+	"io/ioutil"
 	"net"
+	"os"
+	"strings"
 	"syscall"
 
 	"github.com/containernetworking/cni/pkg/skel"
 	"github.com/containernetworking/cni/pkg/types"
-	"github.com/containernetworking/cni/pkg/types/current"
+	"github.com/containernetworking/cni/pkg/types/020"
+	"github.com/containernetworking/cni/pkg/types/040"
+	"github.com/containernetworking/cni/pkg/types/100"
 	"github.com/containernetworking/plugins/pkg/ns"
 	"github.com/containernetworking/plugins/pkg/testutils"
 
@@ -45,7 +50,7 @@ type Net struct {
 	IPAM          *allocator.IPAMConfig  `json:"ipam"`
 	DNS           types.DNS              `json:"dns"`
 	RawPrevResult map[string]interface{} `json:"prevResult,omitempty"`
-	PrevResult    current.Result         `json:"-"`
+	PrevResult    types100.Result        `json:"-"`
 }
 
 func buildOneConfig(netName string, cniVersion string, orig *Net, prevResult types.Result) (*Net, error) {
@@ -91,14 +96,90 @@ func buildOneConfig(netName string, cniVersion string, orig *Net, prevResult typ
 
 }
 
+type tester interface {
+	// verifyResult minimally verifies the Result and returns the interface's MAC address
+	verifyResult(result types.Result, name string) string
+}
+
+type testerBase struct{}
+
+type testerV10x testerBase
+type testerV04x testerBase
+type testerV03x testerBase
+type testerV01xOr02x testerBase
+
+func newTesterByVersion(version string) tester {
+	switch {
+	case strings.HasPrefix(version, "1.0."):
+		return &testerV10x{}
+	case strings.HasPrefix(version, "0.4."):
+		return &testerV04x{}
+	case strings.HasPrefix(version, "0.3."):
+		return &testerV03x{}
+	default:
+		return &testerV01xOr02x{}
+	}
+}
+
+// verifyResult minimally verifies the Result and returns the interface's MAC address
+func (t *testerV10x) verifyResult(result types.Result, name string) string {
+	r, err := types100.GetResult(result)
+	Expect(err).NotTo(HaveOccurred())
+
+	Expect(len(r.Interfaces)).To(Equal(1))
+	Expect(r.Interfaces[0].Name).To(Equal(name))
+	Expect(len(r.IPs)).To(Equal(1))
+
+	return r.Interfaces[0].Mac
+}
+
+func verify0403(result types.Result, name string) string {
+	r, err := types040.GetResult(result)
+	Expect(err).NotTo(HaveOccurred())
+
+	Expect(len(r.Interfaces)).To(Equal(1))
+	Expect(r.Interfaces[0].Name).To(Equal(name))
+	Expect(len(r.IPs)).To(Equal(1))
+
+	return r.Interfaces[0].Mac
+}
+
+// verifyResult minimally verifies the Result and returns the interface's MAC address
+func (t *testerV04x) verifyResult(result types.Result, name string) string {
+	return verify0403(result, name)
+}
+
+// verifyResult minimally verifies the Result and returns the interface's MAC address
+func (t *testerV03x) verifyResult(result types.Result, name string) string {
+	return verify0403(result, name)
+}
+
+// verifyResult minimally verifies the Result and returns the interface's MAC address
+func (t *testerV01xOr02x) verifyResult(result types.Result, name string) string {
+	r, err := types020.GetResult(result)
+	Expect(err).NotTo(HaveOccurred())
+
+	Expect(r.IP4.IP.IP).NotTo(BeNil())
+	Expect(r.IP6).To(BeNil())
+
+	// 0.2 and earlier don't return MAC address
+	return ""
+}
+
 var _ = Describe("vlan Operations", func() {
-	var originalNS ns.NetNS
+	var originalNS, targetNS ns.NetNS
+	var dataDir string
 
 	BeforeEach(func() {
 		// Create a new NetNS so we don't modify the host
 		var err error
 		originalNS, err = testutils.NewNS()
 		Expect(err).NotTo(HaveOccurred())
+		targetNS, err = testutils.NewNS()
+		Expect(err).NotTo(HaveOccurred())
+
+		dataDir, err = ioutil.TempDir("", "vlan_test")
+		Expect(err).NotTo(HaveOccurred())
 
 		err = originalNS.Do(func(ns.NetNS) error {
 			defer GinkgoRecover()
@@ -120,376 +201,290 @@ var _ = Describe("vlan Operations", func() {
 	})
 
 	AfterEach(func() {
+		Expect(os.RemoveAll(dataDir)).To(Succeed())
 		Expect(originalNS.Close()).To(Succeed())
 		Expect(testutils.UnmountNS(originalNS)).To(Succeed())
+		Expect(targetNS.Close()).To(Succeed())
+		Expect(testutils.UnmountNS(targetNS)).To(Succeed())
 	})
 
-	It("creates an vlan link in a non-default namespace with given MTU", func() {
-		conf := &NetConf{
-			NetConf: types.NetConf{
-				CNIVersion: "0.3.0",
-				Name:       "testConfig",
-				Type:       "vlan",
-			},
-			Master: MASTER_NAME,
-			VlanId: 33,
-			MTU:    1500,
-		}
+	for _, ver := range testutils.AllSpecVersions {
+		// Redefine ver inside for scope so real value is picked up by each dynamically defined It()
+		// See Gingkgo's "Patterns for dynamically generating tests" documentation.
+		ver := ver
 
-		// Create vlan in other namespace
-		targetNs, err := testutils.NewNS()
-		Expect(err).NotTo(HaveOccurred())
-		defer targetNs.Close()
+		It(fmt.Sprintf("[%s] creates an vlan link in a non-default namespace with given MTU", ver), func() {
+			conf := &NetConf{
+				NetConf: types.NetConf{
+					CNIVersion: ver,
+					Name:       "testConfig",
+					Type:       "vlan",
+				},
+				Master: MASTER_NAME,
+				VlanId: 33,
+				MTU:    1500,
+			}
 
-		err = originalNS.Do(func(ns.NetNS) error {
-			defer GinkgoRecover()
+			// Create vlan in other namespace
+			err := originalNS.Do(func(ns.NetNS) error {
+				defer GinkgoRecover()
 
-			_, err := createVlan(conf, "foobar0", targetNs)
-			Expect(err).NotTo(HaveOccurred())
-			return nil
-		})
-
-		Expect(err).NotTo(HaveOccurred())
-
-		// Make sure vlan link exists in the target namespace
-		err = targetNs.Do(func(ns.NetNS) error {
-			defer GinkgoRecover()
-
-			link, err := netlink.LinkByName("foobar0")
-			Expect(err).NotTo(HaveOccurred())
-			Expect(link.Attrs().Name).To(Equal("foobar0"))
-			Expect(link.Attrs().MTU).To(Equal(1500))
-			return nil
-		})
-		Expect(err).NotTo(HaveOccurred())
-	})
-
-	It("creates an vlan link in a non-default namespace with master's MTU", func() {
-		conf := &NetConf{
-			NetConf: types.NetConf{
-				CNIVersion: "0.3.0",
-				Name:       "testConfig",
-				Type:       "vlan",
-			},
-			Master: MASTER_NAME,
-			VlanId: 33,
-		}
-
-		// Create vlan in other namespace
-		targetNs, err := testutils.NewNS()
-		Expect(err).NotTo(HaveOccurred())
-		defer targetNs.Close()
-
-		err = originalNS.Do(func(ns.NetNS) error {
-			defer GinkgoRecover()
-
-			m, err := netlink.LinkByName(MASTER_NAME)
-			Expect(err).NotTo(HaveOccurred())
-			err = netlink.LinkSetMTU(m, 1200)
-			Expect(err).NotTo(HaveOccurred())
-
-			_, err = createVlan(conf, "foobar0", targetNs)
-			Expect(err).NotTo(HaveOccurred())
-			return nil
-		})
-
-		Expect(err).NotTo(HaveOccurred())
-
-		// Make sure vlan link exists in the target namespace
-		err = targetNs.Do(func(ns.NetNS) error {
-			defer GinkgoRecover()
-
-			link, err := netlink.LinkByName("foobar0")
-			Expect(err).NotTo(HaveOccurred())
-			Expect(link.Attrs().Name).To(Equal("foobar0"))
-			Expect(link.Attrs().MTU).To(Equal(1200))
-			return nil
-		})
-		Expect(err).NotTo(HaveOccurred())
-	})
-
-	It("configures and deconfigures an vlan link with ADD/DEL", func() {
-		const IFNAME = "eth0"
-
-		conf := fmt.Sprintf(`{
-    "cniVersion": "0.3.0",
-    "name": "mynet",
-    "type": "vlan",
-    "master": "%s",
-    "ipam": {
-        "type": "host-local",
-        "subnet": "10.1.2.0/24"
-    }
-}`, MASTER_NAME)
-
-		targetNs, err := testutils.NewNS()
-		Expect(err).NotTo(HaveOccurred())
-		defer targetNs.Close()
-
-		args := &skel.CmdArgs{
-			ContainerID: "dummy",
-			Netns:       targetNs.Path(),
-			IfName:      IFNAME,
-			StdinData:   []byte(conf),
-		}
-
-		var result *current.Result
-		err = originalNS.Do(func(ns.NetNS) error {
-			defer GinkgoRecover()
-
-			r, _, err := testutils.CmdAddWithArgs(args, func() error {
-				return cmdAdd(args)
+				_, err := createVlan(conf, "foobar0", targetNS)
+				Expect(err).NotTo(HaveOccurred())
+				return nil
 			})
 			Expect(err).NotTo(HaveOccurred())
 
-			result, err = current.GetResult(r)
-			Expect(err).NotTo(HaveOccurred())
+			// Make sure vlan link exists in the target namespace
+			err = targetNS.Do(func(ns.NetNS) error {
+				defer GinkgoRecover()
 
-			Expect(len(result.Interfaces)).To(Equal(1))
-			Expect(result.Interfaces[0].Name).To(Equal(IFNAME))
-			Expect(len(result.IPs)).To(Equal(1))
-			return nil
-		})
-		Expect(err).NotTo(HaveOccurred())
-
-		// Make sure vlan link exists in the target namespace
-		err = targetNs.Do(func(ns.NetNS) error {
-			defer GinkgoRecover()
-
-			link, err := netlink.LinkByName(IFNAME)
-			Expect(err).NotTo(HaveOccurred())
-			Expect(link.Attrs().Name).To(Equal(IFNAME))
-
-			hwaddr, err := net.ParseMAC(result.Interfaces[0].Mac)
-			Expect(err).NotTo(HaveOccurred())
-			Expect(link.Attrs().HardwareAddr).To(Equal(hwaddr))
-
-			addrs, err := netlink.AddrList(link, syscall.AF_INET)
-			Expect(err).NotTo(HaveOccurred())
-			Expect(len(addrs)).To(Equal(1))
-			return nil
-		})
-		Expect(err).NotTo(HaveOccurred())
-
-		err = originalNS.Do(func(ns.NetNS) error {
-			defer GinkgoRecover()
-
-			err = testutils.CmdDelWithArgs(args, func() error {
-				return cmdDel(args)
+				link, err := netlink.LinkByName("foobar0")
+				Expect(err).NotTo(HaveOccurred())
+				Expect(link.Attrs().Name).To(Equal("foobar0"))
+				Expect(link.Attrs().MTU).To(Equal(1500))
+				return nil
 			})
 			Expect(err).NotTo(HaveOccurred())
-			return nil
 		})
-		Expect(err).NotTo(HaveOccurred())
 
-		// Make sure vlan link has been deleted
-		err = targetNs.Do(func(ns.NetNS) error {
-			defer GinkgoRecover()
+		It(fmt.Sprintf("[%s] creates an vlan link in a non-default namespace with master's MTU", ver), func() {
+			conf := &NetConf{
+				NetConf: types.NetConf{
+					CNIVersion: ver,
+					Name:       "testConfig",
+					Type:       "vlan",
+				},
+				Master: MASTER_NAME,
+				VlanId: 33,
+			}
 
-			link, err := netlink.LinkByName(IFNAME)
-			Expect(err).To(HaveOccurred())
-			Expect(link).To(BeNil())
-			return nil
-		})
-		Expect(err).NotTo(HaveOccurred())
+			// Create vlan in other namespace
+			err := originalNS.Do(func(ns.NetNS) error {
+				defer GinkgoRecover()
 
-		// DEL can be called multiple times, make sure no error is returned
-		// if the device is already removed.
-		err = originalNS.Do(func(ns.NetNS) error {
-			defer GinkgoRecover()
+				m, err := netlink.LinkByName(MASTER_NAME)
+				Expect(err).NotTo(HaveOccurred())
+				err = netlink.LinkSetMTU(m, 1200)
+				Expect(err).NotTo(HaveOccurred())
 
-			err = testutils.CmdDelWithArgs(args, func() error {
-				return cmdDel(args)
-			})
-			Expect(err).NotTo(HaveOccurred())
-			return nil
-		})
-		Expect(err).NotTo(HaveOccurred())
-	})
-
-	It("configures and deconfigures an CNI V4 vlan link with ADD/CHECK/DEL", func() {
-		const IFNAME = "eth0"
-
-		conf := fmt.Sprintf(`{
-    "cniVersion": "0.4.0",
-    "name": "vlanTestv4",
-    "type": "vlan",
-    "master": "%s",
-    "vlanId": 1234,
-    "ipam": {
-        "type": "host-local",
-        "subnet": "10.1.2.0/24"
-    }
-}`, MASTER_NAME)
-
-		targetNs, err := testutils.NewNS()
-		Expect(err).NotTo(HaveOccurred())
-		defer targetNs.Close()
-
-		args := &skel.CmdArgs{
-			ContainerID: "dummy",
-			Netns:       targetNs.Path(),
-			IfName:      IFNAME,
-			StdinData:   []byte(conf),
-		}
-
-		var result *current.Result
-		err = originalNS.Do(func(ns.NetNS) error {
-			defer GinkgoRecover()
-
-			r, _, err := testutils.CmdAddWithArgs(args, func() error {
-				return cmdAdd(args)
+				_, err = createVlan(conf, "foobar0", targetNS)
+				Expect(err).NotTo(HaveOccurred())
+				return nil
 			})
 			Expect(err).NotTo(HaveOccurred())
 
-			result, err = current.GetResult(r)
-			Expect(err).NotTo(HaveOccurred())
+			// Make sure vlan link exists in the target namespace
+			err = targetNS.Do(func(ns.NetNS) error {
+				defer GinkgoRecover()
 
-			Expect(len(result.Interfaces)).To(Equal(1))
-			Expect(result.Interfaces[0].Name).To(Equal(IFNAME))
-			Expect(len(result.IPs)).To(Equal(1))
-			return nil
-		})
-		Expect(err).NotTo(HaveOccurred())
-
-		// Make sure vlan link exists in the target namespace
-		err = targetNs.Do(func(ns.NetNS) error {
-			defer GinkgoRecover()
-
-			link, err := netlink.LinkByName(IFNAME)
-			Expect(err).NotTo(HaveOccurred())
-			Expect(link.Attrs().Name).To(Equal(IFNAME))
-
-			hwaddr, err := net.ParseMAC(result.Interfaces[0].Mac)
-			Expect(err).NotTo(HaveOccurred())
-			Expect(link.Attrs().HardwareAddr).To(Equal(hwaddr))
-
-			addrs, err := netlink.AddrList(link, syscall.AF_INET)
-			Expect(err).NotTo(HaveOccurred())
-			Expect(len(addrs)).To(Equal(1))
-			return nil
-		})
-		Expect(err).NotTo(HaveOccurred())
-
-		// call CmdCheck
-		n := &Net{}
-		err = json.Unmarshal([]byte(conf), &n)
-		Expect(err).NotTo(HaveOccurred())
-
-		n.IPAM, _, err = allocator.LoadIPAMConfig([]byte(conf), "")
-		Expect(err).NotTo(HaveOccurred())
-
-		cniVersion := "0.4.0"
-		newConf, err := buildOneConfig("vlanTestv4", cniVersion, n, result)
-		Expect(err).NotTo(HaveOccurred())
-
-		confString, err := json.Marshal(newConf)
-		Expect(err).NotTo(HaveOccurred())
-
-		args.StdinData = confString
-
-		// CNI Check host-device in the target namespace
-		err = originalNS.Do(func(ns.NetNS) error {
-			defer GinkgoRecover()
-			var err error
-			err = testutils.CmdCheckWithArgs(args, func() error { return cmdCheck(args) })
-			return err
-		})
-		Expect(err).NotTo(HaveOccurred())
-
-		args.StdinData = []byte(conf)
-
-		err = originalNS.Do(func(ns.NetNS) error {
-			defer GinkgoRecover()
-
-			err = testutils.CmdDelWithArgs(args, func() error {
-				return cmdDel(args)
+				link, err := netlink.LinkByName("foobar0")
+				Expect(err).NotTo(HaveOccurred())
+				Expect(link.Attrs().Name).To(Equal("foobar0"))
+				Expect(link.Attrs().MTU).To(Equal(1200))
+				return nil
 			})
 			Expect(err).NotTo(HaveOccurred())
-			return nil
 		})
-		Expect(err).NotTo(HaveOccurred())
 
-		// Make sure vlan link has been deleted
-		err = targetNs.Do(func(ns.NetNS) error {
-			defer GinkgoRecover()
+		It(fmt.Sprintf("[%s] configures and deconfigures a vlan link with ADD/CHECK/DEL", ver), func() {
+			const IFNAME = "eth0"
 
-			link, err := netlink.LinkByName(IFNAME)
-			Expect(err).To(HaveOccurred())
-			Expect(link).To(BeNil())
-			return nil
-		})
-		Expect(err).NotTo(HaveOccurred())
-	})
+			conf := fmt.Sprintf(`{
+			    "cniVersion": "%s",
+			    "name": "vlanTestv4",
+			    "type": "vlan",
+			    "master": "%s",
+			    "vlanId": 1234,
+			    "ipam": {
+				"type": "host-local",
+				"subnet": "10.1.2.0/24",
+				"dataDir": "%s"
+			    }
+			}`, ver, MASTER_NAME, dataDir)
+
+			args := &skel.CmdArgs{
+				ContainerID: "dummy",
+				Netns:       targetNS.Path(),
+				IfName:      IFNAME,
+				StdinData:   []byte(conf),
+			}
+
+			t := newTesterByVersion(ver)
+
+			var result types.Result
+			var macAddress string
+			err := originalNS.Do(func(ns.NetNS) error {
+				defer GinkgoRecover()
+
+				var err error
+				result, _, err = testutils.CmdAddWithArgs(args, func() error {
+					return cmdAdd(args)
+				})
+				Expect(err).NotTo(HaveOccurred())
+
+				macAddress = t.verifyResult(result, IFNAME)
+				return nil
+			})
+			Expect(err).NotTo(HaveOccurred())
+
+			// Make sure vlan link exists in the target namespace
+			err = targetNS.Do(func(ns.NetNS) error {
+				defer GinkgoRecover()
+
+				link, err := netlink.LinkByName(IFNAME)
+				Expect(err).NotTo(HaveOccurred())
+				Expect(link.Attrs().Name).To(Equal(IFNAME))
+
+				if macAddress != "" {
+					hwaddr, err := net.ParseMAC(macAddress)
+					Expect(err).NotTo(HaveOccurred())
+					Expect(link.Attrs().HardwareAddr).To(Equal(hwaddr))
+				}
+
+				addrs, err := netlink.AddrList(link, syscall.AF_INET)
+				Expect(err).NotTo(HaveOccurred())
+				Expect(len(addrs)).To(Equal(1))
+				return nil
+			})
+			Expect(err).NotTo(HaveOccurred())
+
+			// call CmdCheck
+			n := &Net{}
+			err = json.Unmarshal([]byte(conf), &n)
+			Expect(err).NotTo(HaveOccurred())
+
+			n.IPAM, _, err = allocator.LoadIPAMConfig([]byte(conf), "")
+			Expect(err).NotTo(HaveOccurred())
+
+			newConf, err := buildOneConfig("vlanTestv4", ver, n, result)
+			Expect(err).NotTo(HaveOccurred())
+
+			confString, err := json.Marshal(newConf)
+			Expect(err).NotTo(HaveOccurred())
+
+			args.StdinData = confString
+
+			// CNI Check host-device in the target namespace
+			err = originalNS.Do(func(ns.NetNS) error {
+				defer GinkgoRecover()
+				return testutils.CmdCheckWithArgs(args, func() error { return cmdCheck(args) })
+			})
+			if testutils.SpecVersionHasCHECK(ver) {
+				Expect(err).NotTo(HaveOccurred())
+			} else {
+				Expect(err).To(MatchError("config version does not allow CHECK"))
+			}
+
+			args.StdinData = []byte(conf)
 
-	Describe("fails to create vlan link with invalid MTU", func() {
-		conf := `{
-    "cniVersion": "0.3.1",
-    "name": "mynet",
-    "type": "vlan",
-    "master": "%s",
-    "mtu": %d,
-    "ipam": {
-        "type": "host-local",
-        "subnet": "10.1.2.0/24"
-    }
-}`
-		BeforeEach(func() {
-			var err error
 			err = originalNS.Do(func(ns.NetNS) error {
 				defer GinkgoRecover()
 
-				// set master link's MTU to 1500
-				link, err := netlink.LinkByName(MASTER_NAME)
-				Expect(err).NotTo(HaveOccurred())
-				err = netlink.LinkSetMTU(link, 1500)
+				err = testutils.CmdDelWithArgs(args, func() error {
+					return cmdDel(args)
+				})
 				Expect(err).NotTo(HaveOccurred())
+				return nil
+			})
+			Expect(err).NotTo(HaveOccurred())
 
+			// Make sure vlan link has been deleted
+			err = targetNS.Do(func(ns.NetNS) error {
+				defer GinkgoRecover()
+
+				link, err := netlink.LinkByName(IFNAME)
+				Expect(err).To(HaveOccurred())
+				Expect(link).To(BeNil())
+				return nil
+			})
+			Expect(err).NotTo(HaveOccurred())
+
+			// DEL can be called multiple times, make sure no error is returned
+			// if the device is already removed.
+			err = originalNS.Do(func(ns.NetNS) error {
+				defer GinkgoRecover()
+
+				err = testutils.CmdDelWithArgs(args, func() error {
+					return cmdDel(args)
+				})
+				Expect(err).NotTo(HaveOccurred())
 				return nil
 			})
 			Expect(err).NotTo(HaveOccurred())
 		})
 
-		It("fails to create vlan link with greater MTU than master interface", func() {
-			var err error
+		Describe("fails to create vlan link with invalid MTU", func() {
+			const confFmt = `{
+			    "cniVersion": "%s",
+			    "name": "mynet",
+			    "type": "vlan",
+			    "master": "%s",
+			    "mtu": %d,
+			    "ipam": {
+				"type": "host-local",
+				"subnet": "10.1.2.0/24",
+				"dataDir": "%s"
+			    }
+			}`
 
-			args := &skel.CmdArgs{
-				ContainerID: "dummy",
-				Netns:       "/var/run/netns/test",
-				IfName:      "eth0",
-				StdinData:   []byte(fmt.Sprintf(conf, MASTER_NAME, 1600)),
-			}
+			BeforeEach(func() {
+				var err error
+				err = originalNS.Do(func(ns.NetNS) error {
+					defer GinkgoRecover()
 
-			_ = originalNS.Do(func(netNS ns.NetNS) error {
-				defer GinkgoRecover()
+					// set master link's MTU to 1500
+					link, err := netlink.LinkByName(MASTER_NAME)
+					Expect(err).NotTo(HaveOccurred())
+					err = netlink.LinkSetMTU(link, 1500)
+					Expect(err).NotTo(HaveOccurred())
 
-				_, _, err = testutils.CmdAddWithArgs(args, func() error {
-					return cmdAdd(args)
+					return nil
+				})
+				Expect(err).NotTo(HaveOccurred())
+			})
+
+			It(fmt.Sprintf("[%s] fails to create vlan link with greater MTU than master interface", ver), func() {
+				var err error
+
+				args := &skel.CmdArgs{
+					ContainerID: "dummy",
+					Netns:       "/var/run/netns/test",
+					IfName:      "eth0",
+					StdinData:   []byte(fmt.Sprintf(confFmt, ver, MASTER_NAME, 1600, dataDir)),
+				}
+
+				_ = originalNS.Do(func(netNS ns.NetNS) error {
+					defer GinkgoRecover()
+
+					_, _, err = testutils.CmdAddWithArgs(args, func() error {
+						return cmdAdd(args)
+					})
+					Expect(err).To(Equal(fmt.Errorf("invalid MTU 1600, must be [0, master MTU(1500)]")))
+					return nil
+				})
+			})
+
+			It(fmt.Sprintf("[%s] fails to create vlan link with negative MTU", ver), func() {
+				var err error
+
+				args := &skel.CmdArgs{
+					ContainerID: "dummy",
+					Netns:       "/var/run/netns/test",
+					IfName:      "eth0",
+					StdinData:   []byte(fmt.Sprintf(confFmt, ver, MASTER_NAME, -100, dataDir)),
+				}
+
+				_ = originalNS.Do(func(netNS ns.NetNS) error {
+					defer GinkgoRecover()
+
+					_, _, err = testutils.CmdAddWithArgs(args, func() error {
+						return cmdAdd(args)
+					})
+					Expect(err).To(Equal(fmt.Errorf("invalid MTU -100, must be [0, master MTU(1500)]")))
+					return nil
 				})
-				Expect(err).To(Equal(fmt.Errorf("invalid MTU 1600, must be [0, master MTU(1500)]")))
-				return nil
 			})
 		})
-
-		It("fails to create vlan link with negative MTU", func() {
-			var err error
-
-			args := &skel.CmdArgs{
-				ContainerID: "dummy",
-				Netns:       "/var/run/netns/test",
-				IfName:      "eth0",
-				StdinData:   []byte(fmt.Sprintf(conf, MASTER_NAME, -100)),
-			}
-
-			_ = originalNS.Do(func(netNS ns.NetNS) error {
-				defer GinkgoRecover()
-
-				_, _, err = testutils.CmdAddWithArgs(args, func() error {
-					return cmdAdd(args)
-				})
-				Expect(err).To(Equal(fmt.Errorf("invalid MTU -100, must be [0, master MTU(1500)]")))
-				return nil
-			})
-		})
-	})
+	}
 })

plugins/main/windows/win-bridge/README.md

@@ -1,59 +1,5 @@
-# win-bridge plugin
 
-## Overview
+This document has moved to the [containernetworking/cni.dev](https://github.com/containernetworking/cni.dev) repo.
 
-With win-bridge plugin, all containers (on the same host) are plugged into an L2Bridge network that has one endpoint in the host namespace.
+You can find it online here: https://cni.dev/plugins/current/main/win-bridge/
 
-## Example configuration
-```
-{
-	"name": "mynet",
-	"type": "win-bridge",
-	"ipMasqNetwork": "10.244.0.0/16",
-	"ipam": {
-		"type": "host-local",
-		"subnet": "10.10.0.0/16"
-	},
-    "policies":[
-        {
-            "name":"EndpointPolicy",
-            "value":{
-                "Type":"ROUTE",
-                "DestinationPrefix":"10.137.198.27/32",
-                "NeedEncap":true
-            }
-        } 
-    ],
-    "HcnPolicyArgs": [
-        {
-            "Type": "SDNRoute"
-            "Settings": {
-                "DestinationPrefix": "11.0.0.0/8",
-                "NeedEncap": true
-            }
-        }
-    ],
-    "loopbackDSR": true,
-    "capabilities": {
-        "dns": true,
-        "portMappings":  true
-    }
-}
-```
-
-## Network configuration reference
-
-* `ApiVersion` (integer, optional): ApiVersion to use, will default to hns. If set to "2" will try to use hcn APIs.
-* `name` (string, required): the name of the network.
-* `type` (string, required): "win-bridge".
-* `ipMasqNetwork` (string, optional): setup NAT if not empty.
-* `dns` (dictionary, optional): dns config to be used.
- * `Nameservers` (list, optional): list of strings to be used for dns nameservers.
- * `Search` (list, optional): list of stings to be used for dns search.
-* `ipam` (dictionary, optional): IPAM configuration to be used for this network.
-* `Policies` (list, optional): List of hns policies to be used (only used when ApiVersion is < 2).
-* `HcnPolicyArgs` (list, optional): List of hcn policies to be used (only used when ApiVersion is 2).
-* `loopbackDSR` (bool, optional): If true, will add a policy to allow the interface to support loopback direct server return.
-* `capabilities` (dictionary, optional): Runtime capabilities to enable.
- * `dns` (boolean, optional): If true, will take the dns config supplied by the runtime and override other settings.
- * `portMappings` (boolean, optional): If true, will handle HostPort<>ContainerPort mapping using NAT HNS Policies

plugins/main/windows/win-bridge/sample-v1.conf

@@ -0,0 +1,42 @@
+{
+  "name": "cbr0",
+  "type": "win-bridge",
+  "dns": {
+    "nameservers": [
+      "11.0.0.10"
+    ],
+    "search": [
+      "svc.cluster.local"
+    ]
+  },
+  "policies": [
+    {
+      "name": "EndpointPolicy",
+      "value": {
+        "Type": "OutBoundNAT",
+        "ExceptionList": [
+          "192.168.0.0/16",
+          "11.0.0.0/8",
+          "10.137.196.0/23"
+        ]
+      }
+    },
+    {
+      "name": "EndpointPolicy",
+      "value": {
+        "Type": "ROUTE",
+        "DestinationPrefix": "11.0.0.0/8",
+        "NeedEncap": true
+      }
+    },
+    {
+      "name": "EndpointPolicy",
+      "value": {
+        "Type": "ROUTE",
+        "DestinationPrefix": "10.137.198.27/32",
+        "NeedEncap": true
+      }
+    }
+  ],
+  "loopbackDSR": true
+}

plugins/main/windows/win-bridge/sample-v2.conf

@@ -0,0 +1,52 @@
+{
+    "name":"cbr0",
+    "type":"flannel",
+    "delegate":{
+        "apiVersion":2,
+        "type":"win-bridge",
+        "dns":{
+            "nameservers":[
+                "11.0.0.10"
+            ],
+            "search":[
+                "svc.cluster.local"
+            ]
+        },
+        "policies":[
+            {
+                "name":"EndpointPolicy",
+                "value":{
+                    "Type":"OutBoundNAT",
+                    "Settings":{
+                        "Exceptions":[
+                            "192.168.0.0/16",
+                            "11.0.0.0/8",
+                            "10.137.196.0/23"
+                        ]
+                    }
+                }
+            },
+            {
+                "name":"EndpointPolicy",
+                "value":{
+                    "Type":"SDNRoute",
+                    "Settings":{
+                        "DestinationPrefix":"11.0.0.0/8",
+                        "NeedEncap":true
+                    }
+                }
+            },
+            {
+                "name":"EndpointPolicy",
+                "value":{
+                    "Type":"SDNRoute",
+                    "Settings":{
+                        "DestinationPrefix":"10.137.198.27/32",
+                        "NeedEncap":true
+                    }
+                }
+            }
+        ],
+        "loopbackDSR":true
+    }
+}

plugins/main/windows/win-bridge/sample.conf

@@ -1,45 +0,0 @@
-{
-    "name":"cbr0",
-    "type":"flannel",
-    "delegate":{
-        "type":"win-bridge",
-        "dns":{
-            "nameservers":[
-                "11.0.0.10"
-            ],
-            "search":[
-                "svc.cluster.local"
-            ]
-        },
-        "policies":[
-            {
-                "name":"EndpointPolicy",
-                "value":{
-                    "Type":"OutBoundNAT",
-                    "ExceptionList":[
-                        "192.168.0.0/16",
-                        "11.0.0.0/8",
-                        "10.137.196.0/23"
-                    ]
-                }
-            },
-            {
-                "name":"EndpointPolicy",
-                "value":{
-                    "Type":"ROUTE",
-                    "DestinationPrefix":"11.0.0.0/8",
-                    "NeedEncap":true
-                }
-            },
-            {
-                "name":"EndpointPolicy",
-                "value":{
-                    "Type":"ROUTE",
-                    "DestinationPrefix":"10.137.198.27/32",
-                    "NeedEncap":true
-                }
-            }
-        ],
-        "loopbackDSR": true
-    }
-}

plugins/main/windows/win-bridge/win-bridge_windows.go

@@ -25,7 +25,7 @@ import (
 
 	"github.com/containernetworking/cni/pkg/skel"
 	"github.com/containernetworking/cni/pkg/types"
-	"github.com/containernetworking/cni/pkg/types/current"
+	current "github.com/containernetworking/cni/pkg/types/100"
 	"github.com/containernetworking/cni/pkg/version"
 
 	"github.com/containernetworking/plugins/pkg/errors"
@@ -55,21 +55,22 @@ func loadNetConf(bytes []byte) (*NetConf, string, error) {
 	return n, n.CNIVersion, nil
 }
 
-func ProcessEndpointArgs(args *skel.CmdArgs, n *NetConf) (*hns.EndpointInfo, error) {
+func processEndpointArgs(args *skel.CmdArgs, n *NetConf) (*hns.EndpointInfo, error) {
 	epInfo := new(hns.EndpointInfo)
 	epInfo.NetworkName = n.Name
 	epInfo.EndpointName = hns.ConstructEndpointName(args.ContainerID, args.Netns, epInfo.NetworkName)
-	// It's not necessary to have have an IPAM in windows as hns can provide IP/GW
+
+	// it's not necessary to have have an IPAM in windows as HNS can provide IP/GW
 	if n.IPAM.Type != "" {
 		r, err := ipam.ExecAdd(n.IPAM.Type, args.StdinData)
 		if err != nil {
-			return nil, errors.Annotatef(err, "error while ipam.ExecAdd")
+			return nil, errors.Annotatef(err, "error while executing IPAM addition")
 		}
 
-		// Convert whatever the IPAM result was into the current Result type
+		// convert whatever the IPAM result was into the current result
 		result, err := current.NewResultFromResult(r)
 		if err != nil {
-			return nil, errors.Annotatef(err, "error while NewResultFromResult")
+			return nil, errors.Annotatef(err, "error while converting the result from IPAM addition")
 		} else {
 			if len(result.IPs) == 0 {
 				return nil, fmt.Errorf("IPAM plugin return is missing IP config")
@@ -81,12 +82,11 @@ func ProcessEndpointArgs(args *skel.CmdArgs, n *NetConf) (*hns.EndpointInfo, err
 			epInfo.Gateway[len(epInfo.Gateway)-1] += 2
 		}
 	}
-	// NAT based on the the configured cluster network
-	if len(n.IPMasqNetwork) != 0 {
-		n.ApplyOutboundNatPolicy(n.IPMasqNetwork)
-	}
 
-	// Add HostPort mapping if any present
+	// configure sNAT exception
+	n.ApplyOutboundNatPolicy(n.IPMasqNetwork)
+
+	// add port mapping if any present
 	n.ApplyPortMappingPolicy(n.RuntimeConfig.PortMaps)
 
 	epInfo.DNS = n.GetDNS()
@@ -98,39 +98,37 @@ func cmdHnsAdd(args *skel.CmdArgs, n *NetConf) (*current.Result, error) {
 	networkName := n.Name
 	hnsNetwork, err := hcsshim.GetHNSNetworkByName(networkName)
 	if err != nil {
-		return nil, errors.Annotatef(err, "error while GETHNSNewtorkByName(%s)", networkName)
+		return nil, errors.Annotatef(err, "error while getting network %v", networkName)
 	}
-
 	if hnsNetwork == nil {
-		return nil, fmt.Errorf("network %v not found", networkName)
+		return nil, fmt.Errorf("network %v is not found", networkName)
 	}
-
 	if !strings.EqualFold(hnsNetwork.Type, "L2Bridge") && !strings.EqualFold(hnsNetwork.Type, "L2Tunnel") {
-		return nil, fmt.Errorf("network %v is of an unexpected type: %v", networkName, hnsNetwork.Type)
+		return nil, fmt.Errorf("network %v is of unexpected type: %v", networkName, hnsNetwork.Type)
 	}
 
 	epName := hns.ConstructEndpointName(args.ContainerID, args.Netns, n.Name)
-	hnsEndpoint, err := hns.ProvisionEndpoint(epName, hnsNetwork.Id, args.ContainerID, args.Netns, func() (*hcsshim.HNSEndpoint, error) {
-		epInfo, err := ProcessEndpointArgs(args, n)
-		epInfo.NetworkId = hnsNetwork.Id
+	hnsEndpoint, err := hns.AddHnsEndpoint(epName, hnsNetwork.Id, args.ContainerID, args.Netns, func() (*hcsshim.HNSEndpoint, error) {
+		epInfo, err := processEndpointArgs(args, n)
 		if err != nil {
-			return nil, errors.Annotatef(err, "error while ProcessEndpointArgs")
+			return nil, errors.Annotate(err, "error while processing endpoint args")
 		}
+		epInfo.NetworkId = hnsNetwork.Id
+
 		hnsEndpoint, err := hns.GenerateHnsEndpoint(epInfo, &n.NetConf)
 		if err != nil {
-			return nil, errors.Annotatef(err, "error while GenerateHnsEndpoint")
+			return nil, errors.Annotate(err, "error while generating HNSEndpoint")
 		}
 		return hnsEndpoint, nil
 	})
 	if err != nil {
-		return nil, errors.Annotatef(err, "error while ProvisionEndpoint(%v,%v,%v)", epName, hnsNetwork.Id, args.ContainerID)
+		return nil, errors.Annotate(err, "error while adding HNSEndpoint")
 	}
 
-	result, err := hns.ConstructResult(hnsNetwork, hnsEndpoint)
+	result, err := hns.ConstructHnsResult(hnsNetwork, hnsEndpoint)
 	if err != nil {
-		return nil, errors.Annotatef(err, "error while constructResult")
+		return nil, errors.Annotate(err, "error while constructing HNSEndpoint addition result")
 	}
-
 	return result, nil
 }
 
@@ -138,48 +136,44 @@ func cmdHcnAdd(args *skel.CmdArgs, n *NetConf) (*current.Result, error) {
 	networkName := n.Name
 	hcnNetwork, err := hcn.GetNetworkByName(networkName)
 	if err != nil {
-		return nil, errors.Annotatef(err, "error while GetNetworkByName(%s)", networkName)
+		return nil, errors.Annotatef(err, "error while getting network %v", networkName)
 	}
-
 	if hcnNetwork == nil {
-		return nil, fmt.Errorf("network %v not found", networkName)
+		return nil, fmt.Errorf("network %v is not found", networkName)
 	}
-
 	if hcnNetwork.Type != hcn.L2Bridge && hcnNetwork.Type != hcn.L2Tunnel {
 		return nil, fmt.Errorf("network %v is of unexpected type: %v", networkName, hcnNetwork.Type)
 	}
 
 	epName := hns.ConstructEndpointName(args.ContainerID, args.Netns, n.Name)
-
 	hcnEndpoint, err := hns.AddHcnEndpoint(epName, hcnNetwork.Id, args.Netns, func() (*hcn.HostComputeEndpoint, error) {
-		epInfo, err := ProcessEndpointArgs(args, n)
+		epInfo, err := processEndpointArgs(args, n)
 		if err != nil {
-			return nil, errors.Annotatef(err, "error while ProcessEndpointArgs")
+			return nil, errors.Annotate(err, "error while processing endpoint args")
 		}
 		epInfo.NetworkId = hcnNetwork.Id
 
 		hcnEndpoint, err := hns.GenerateHcnEndpoint(epInfo, &n.NetConf)
 		if err != nil {
-			return nil, errors.Annotatef(err, "error while GenerateHcnEndpoint")
+			return nil, errors.Annotate(err, "error while generating HostComputeEndpoint")
 		}
 		return hcnEndpoint, nil
 	})
 	if err != nil {
-		return nil, errors.Annotatef(err, "error while AddHcnEndpoint(%v,%v,%v)", epName, hcnNetwork.Id, args.Netns)
+		return nil, errors.Annotate(err, "error while adding HostComputeEndpoint")
 	}
 
 	result, err := hns.ConstructHcnResult(hcnNetwork, hcnEndpoint)
 	if err != nil {
-		return nil, errors.Annotatef(err, "error while ConstructHcnResult")
+		return nil, errors.Annotate(err, "error while constructing HostComputeEndpoint addition result")
 	}
-
 	return result, nil
 }
 
 func cmdAdd(args *skel.CmdArgs) error {
 	n, cniVersion, err := loadNetConf(args.StdinData)
 	if err != nil {
-		return errors.Annotate(err, "error while loadNetConf")
+		return err
 	}
 
 	var result *current.Result
@@ -188,15 +182,11 @@ func cmdAdd(args *skel.CmdArgs) error {
 	} else {
 		result, err = cmdHnsAdd(args, n)
 	}
-
 	if err != nil {
 		ipam.ExecDel(n.IPAM.Type, args.StdinData)
-		return errors.Annotate(err, "error while executing ADD command")
+		return err
 	}
 
-	if result == nil {
-		return fmt.Errorf("result for ADD not populated correctly")
-	}
 	return types.PrintResult(result, cniVersion)
 }
 
@@ -215,9 +205,8 @@ func cmdDel(args *skel.CmdArgs) error {
 
 	if n.ApiVersion == 2 {
 		return hns.RemoveHcnEndpoint(epName)
-	} else {
-		return hns.DeprovisionEndpoint(epName, args.Netns, args.ContainerID)
 	}
+	return hns.RemoveHnsEndpoint(epName, args.Netns, args.ContainerID)
 }
 
 func cmdCheck(_ *skel.CmdArgs) error {
@@ -226,5 +215,5 @@ func cmdCheck(_ *skel.CmdArgs) error {
 }
 
 func main() {
-	skel.PluginMain(cmdAdd, cmdCheck, cmdDel, version.PluginSupports("0.1.0", "0.2.0", "0.3.0"), bv.BuildString("win-bridge"))
+	skel.PluginMain(cmdAdd, cmdCheck, cmdDel, version.All, bv.BuildString("win-bridge"))
 }

plugins/main/windows/win-overlay/README.md

@@ -1,38 +1,5 @@
-# win-overlay plugin
 
-## Overview
+This document has moved to the [containernetworking/cni.dev](https://github.com/containernetworking/cni.dev) repo.
 
-With win-overlay plugin, all containers (on the same host) are plugged into an Overlay network based on VXLAN encapsulation. 
+You can find it online here: https://cni.dev/plugins/current/main/win-overlay/
 
-## Example configuration
-```
-{
-	"name": "mynet",
-	"type": "win-overlay",
-	"ipMasq": true,
-	"endpointMacPrefix": "0E-2A",
-	"ipam": {
-		"type": "host-local",
-		"subnet": "10.10.0.0/16"
-	},
-    "loopbackDSR": true,
-    "capabilites": {
-        "dns": true
-    }
-}
-```
-
-## Network configuration reference
-
-* `name` (string, required): the name of the network.
-* `type` (string, required): "win-overlay".
-* `ipMasq` (bool, optional): the inverse of `$FLANNEL_IPMASQ`, setup NAT for the hnsNetwork subnet.
-* `dns` (dictionary, optional): dns config to be used.
- * `Nameservers` (list, optional): list of strings to be used for dns nameservers.
- * `Search` (list, optional): list of stings to be used for dns search.
-* `endpointMacPrefix` (string, optional): set to the MAC prefix configured for Flannel.
-* `Policies` (list, optional): List of hns policies to be used.
-* `ipam` (dictionary, required): IPAM configuration to be used for this network.
-* `loopbackDSR` (bool, optional): If true, will add a policy to allow the interface to support loopback direct server return.
-* `capabilities` (dictionary, optional): runtime capabilities to be parsed and injected by runtime.
- * `dns` (boolean, optional): If true, will take the dns config supplied by the runtime and override other settings.

plugins/main/windows/win-overlay/sample.conf

@@ -1,36 +1,33 @@
-{
-    "cniVersion":"0.2.0",
-    "name":"vxlan0",
-    "type":"flannel",
-    "delegate":{
-        "type":"win-overlay",
-        "dns":{
-            "nameservers":[
-                "11.0.0.10"
-            ],
-            "search":[
-                "svc.cluster.local"
-            ]
-        },
-        "policies":[
-            {
-                "name":"EndpointPolicy",
-                "value":{
-                    "Type":"OutBoundNAT",
-                    "ExceptionList":[
-                        "192.168.0.0/16",
-                        "11.0.0.0/8"
-                    ]
-                }
-            },
-            {
-                "name":"EndpointPolicy",
-                "value":{
-                    "Type":"ROUTE",
-                    "DestinationPrefix":"11.0.0.0/8",
-                    "NeedEncap":true
-                }
-            }
-        ]
-    }
-}
+{
+  "cniVersion": "0.2.0",
+  "name": "vxlan0",
+  "type": "win-overlay",
+  "dns": {
+    "nameservers": [
+      "11.0.0.10"
+    ],
+    "search": [
+      "svc.cluster.local"
+    ]
+  },
+  "policies": [
+    {
+      "name": "EndpointPolicy",
+      "value": {
+        "Type": "OutBoundNAT",
+        "ExceptionList": [
+          "192.168.0.0/16",
+          "11.0.0.0/8"
+        ]
+      }
+    },
+    {
+      "name": "EndpointPolicy",
+      "value": {
+        "Type": "ROUTE",
+        "DestinationPrefix": "11.0.0.0/8",
+        "NeedEncap": true
+      }
+    }
+  ]
+}

plugins/main/windows/win-overlay/win-overlay_windows.go

@@ -24,7 +24,7 @@ import (
 
 	"github.com/containernetworking/cni/pkg/skel"
 	"github.com/containernetworking/cni/pkg/types"
-	"github.com/containernetworking/cni/pkg/types/current"
+	current "github.com/containernetworking/cni/pkg/types/100"
 	"github.com/containernetworking/cni/pkg/version"
 
 	"github.com/containernetworking/plugins/pkg/errors"
@@ -86,7 +86,7 @@ func cmdAdd(args *skel.CmdArgs) error {
 
 	epName := hns.ConstructEndpointName(args.ContainerID, args.Netns, n.Name)
 
-	hnsEndpoint, err := hns.ProvisionEndpoint(epName, hnsNetwork.Id, args.ContainerID, args.Netns, func() (*hcsshim.HNSEndpoint, error) {
+	hnsEndpoint, err := hns.AddHnsEndpoint(epName, hnsNetwork.Id, args.ContainerID, args.Netns, func() (*hcsshim.HNSEndpoint, error) {
 		// run the IPAM plugin and get back the config to apply
 		r, err := ipam.ExecAdd(n.IPAM.Type, args.StdinData)
 		if err != nil {
@@ -119,7 +119,7 @@ func cmdAdd(args *skel.CmdArgs) error {
 
 		result.DNS = n.GetDNS()
 		if n.LoopbackDSR {
-			n.ApplyLoopbackDSR(&ipAddr)
+			n.ApplyLoopbackDSRPolicy(&ipAddr)
 		}
 		hnsEndpoint := &hcsshim.HNSEndpoint{
 			Name:           epName,
@@ -129,7 +129,7 @@ func cmdAdd(args *skel.CmdArgs) error {
 			GatewayAddress: gw,
 			IPAddress:      ipAddr,
 			MacAddress:     macAddr,
-			Policies:       n.MarshalPolicies(),
+			Policies:       n.GetHNSEndpointPolicies(),
 		}
 
 		return hnsEndpoint, nil
@@ -140,10 +140,10 @@ func cmdAdd(args *skel.CmdArgs) error {
 		}
 	}()
 	if err != nil {
-		return errors.Annotatef(err, "error while ProvisionEndpoint(%v,%v,%v)", epName, hnsNetwork.Id, args.ContainerID)
+		return errors.Annotatef(err, "error while AddHnsEndpoint(%v,%v,%v)", epName, hnsNetwork.Id, args.ContainerID)
 	}
 
-	result, err := hns.ConstructResult(hnsNetwork, hnsEndpoint)
+	result, err := hns.ConstructHnsResult(hnsNetwork, hnsEndpoint)
 	if err != nil {
 		return errors.Annotatef(err, "error while constructResult")
 	}
@@ -164,7 +164,7 @@ func cmdDel(args *skel.CmdArgs) error {
 
 	epName := hns.ConstructEndpointName(args.ContainerID, args.Netns, n.Name)
 
-	return hns.DeprovisionEndpoint(epName, args.Netns, args.ContainerID)
+	return hns.RemoveHnsEndpoint(epName, args.Netns, args.ContainerID)
 }
 
 func cmdCheck(_ *skel.CmdArgs) error {
@@ -173,5 +173,5 @@ func cmdCheck(_ *skel.CmdArgs) error {
 }
 
 func main() {
-	skel.PluginMain(cmdAdd, cmdCheck, cmdDel, version.PluginSupports("0.1.0", "0.2.0", "0.3.0"), bv.BuildString("win-overlay"))
+	skel.PluginMain(cmdAdd, cmdCheck, cmdDel, version.All, bv.BuildString("win-overlay"))
 }

plugins/meta/bandwidth/README.md

@@ -1,64 +1,5 @@
-# bandwidth plugin
 
-## Overview
+This document has moved to the [containernetworking/cni.dev](https://github.com/containernetworking/cni.dev) repo.
 
-This plugin provides a way to use and configure Linux's Traffic control (tc) subystem. tc encompasses the sets of mechanisms and operations by which packets are queued for transmission/reception on a network interface.
+You can find it online here: https://cni.dev/plugins/current/meta/bandwidth/
 
-This plugin configures a token bucket filter (tbf) queuing discipline (qdisc) on both ingress and egress traffic. Resulting in traffic being shaped when reading / writing.
-
-Due to limitations on tc shaping rules for ingress, this plugin creates an Intermediate Functional Block device (ifb) to redirect packets from the host interface. tc tbf is then applied to the ifb device. The packets that were redirected to the ifb devices, are written OUT (and shaped) to the host interface.
-
-This plugin is only useful when used in addition to other plugins.
-
-## Chaining
-
-The bandwidth plugin applies traffic shaping to interfaces (as described above) created by previously applied plugins.
-
-The following is an example [json configuration list](https://github.com/containernetworking/cni/blob/master/SPEC.md#network-configuration-list-runtime-examples) for creating a `ptp` between the host -> container via veth interfaces, whereby traffic is shaped by the `bandwidth` plugin:
-
-```json
-{
-  "cniVersion": "0.3.1",
-  "name": "mynet",
-  "plugins": [
-    {
-      "type": "ptp",
-      "ipMasq": true,
-      "mtu": 512,
-      "ipam": {
-          "type": "host-local",
-          "subnet": "10.0.0.0/24"
-      },
-      "dns": {
-        "nameservers": [ "10.1.0.1" ]
-      }
-    },
-    {
-      "name": "slowdown",
-      "type": "bandwidth",
-      "ingressRate": 123,
-      "ingressBurst": 456,
-      "egressRate": 123,
-      "egressBurst": 456
-    }
-  ]
-}
-```
-
-The result is an `ifb` device in the host namespace redirecting to the `host-interface`, with `tc tbf` applied on the `ifb` device and the `container-interface`
-
-## Network configuration reference
-* ingressRate: is the rate in bps at which traffic can enter an interface. (See http://man7.org/linux/man-pages/man8/tbf.8.html)
-* ingressBurst: is the maximum amount in bits that tokens can be made available for instantaneously. (See http://man7.org/linux/man-pages/man8/tbf.8.html)
-* egressRate: is the rate in bps at which traffic can leave an interface. (See http://man7.org/linux/man-pages/man8/tbf.8.html)
-* egressBurst: is the maximum amount in bits that tokens can be made available for instantaneously. (See http://man7.org/linux/man-pages/man8/tbf.8.html)
-
-Both ingressRate and ingressBurst must be set in order to limit ingress bandwidth. If neither one is set, then ingress bandwidth is not limited.
-Both egressRate and egressBurst must be set in order to limit egress bandwidth. If neither one is set, then egress bandwidth is not limited.
-
-
-## tc tbf documentation
-
-- [tldp traffic control](http://tldp.org/HOWTO/Traffic-Control-HOWTO/components.html)
-- [man tbf](http://man7.org/linux/man-pages/man8/tbf.8.html)
-- [tc ingress and ifb mirroring](https://serverfault.com/questions/350023/tc-ingress-policing-and-ifb-mirroring)

plugins/meta/bandwidth/bandwidth_suite_test.go

@@ -39,14 +39,18 @@ func TestTBF(t *testing.T) {
 	RunSpecs(t, "plugins/meta/bandwidth")
 }
 
-var echoServerBinaryPath string
+var echoServerBinaryPath, echoClientBinaryPath string
 
 var _ = SynchronizedBeforeSuite(func() []byte {
-	binaryPath, err := gexec.Build("github.com/containernetworking/plugins/pkg/testutils/echosvr")
+	serverBinaryPath, err := gexec.Build("github.com/containernetworking/plugins/pkg/testutils/echo/server")
 	Expect(err).NotTo(HaveOccurred())
-	return []byte(binaryPath)
+	clientBinaryPath, err := gexec.Build("github.com/containernetworking/plugins/pkg/testutils/echo/client")
+	Expect(err).NotTo(HaveOccurred())
+	return []byte(strings.Join([]string{serverBinaryPath, clientBinaryPath}, ","))
 }, func(data []byte) {
-	echoServerBinaryPath = string(data)
+	binaries := strings.Split(string(data), ",")
+	echoServerBinaryPath = binaries[0]
+	echoClientBinaryPath = binaries[1]
 })
 
 var _ = SynchronizedAfterSuite(func() {}, func() {
@@ -84,26 +88,25 @@ func startEchoServerInNamespace(netNS ns.NetNS) (int, *gexec.Session, error) {
 }
 
 func makeTcpClientInNS(netns string, address string, port int, numBytes int) {
-	message := bytes.Repeat([]byte{'a'}, numBytes)
+	payload := bytes.Repeat([]byte{'a'}, numBytes)
+	message := string(payload)
 
-	bin, err := exec.LookPath("nc")
-	Expect(err).NotTo(HaveOccurred())
 	var cmd *exec.Cmd
 	if netns != "" {
 		netns = filepath.Base(netns)
-		cmd = exec.Command("ip", "netns", "exec", netns, bin, "-v", address, strconv.Itoa(port))
+		cmd = exec.Command("ip", "netns", "exec", netns, echoClientBinaryPath, "--target", fmt.Sprintf("%s:%d", address, port), "--message", message)
 	} else {
-		cmd = exec.Command("nc", address, strconv.Itoa(port))
+		cmd = exec.Command(echoClientBinaryPath, "--target", fmt.Sprintf("%s:%d", address, port), "--message", message)
 	}
 	cmd.Stdin = bytes.NewBuffer([]byte(message))
 	cmd.Stderr = GinkgoWriter
 	out, err := cmd.Output()
 
 	Expect(err).NotTo(HaveOccurred())
-	Expect(string(out)).To(Equal(string(message)))
+	Expect(string(out)).To(Equal(message))
 }
 
-func createVeth(hostNamespace string, hostVethIfName string, containerNamespace string, containerVethIfName string, hostIP []byte, containerIP []byte, hostIfaceMTU int) {
+func createVeth(hostNs ns.NetNS, hostVethIfName string, containerNs ns.NetNS, containerVethIfName string, hostIP []byte, containerIP []byte, hostIfaceMTU int) {
 	vethDeviceRequest := &netlink.Veth{
 		LinkAttrs: netlink.LinkAttrs{
 			Name:  hostVethIfName,
@@ -113,10 +116,7 @@ func createVeth(hostNamespace string, hostVethIfName string, containerNamespace
 		PeerName: containerVethIfName,
 	}
 
-	hostNs, err := ns.GetNS(hostNamespace)
-	Expect(err).NotTo(HaveOccurred())
-
-	err = hostNs.Do(func(_ ns.NetNS) error {
+	err := hostNs.Do(func(_ ns.NetNS) error {
 		if err := netlink.LinkAdd(vethDeviceRequest); err != nil {
 			return fmt.Errorf("creating veth pair: %s", err)
 		}
@@ -126,11 +126,6 @@ func createVeth(hostNamespace string, hostVethIfName string, containerNamespace
 			return fmt.Errorf("failed to find newly-created veth device %q: %v", containerVethIfName, err)
 		}
 
-		containerNs, err := ns.GetNS(containerNamespace)
-		if err != nil {
-			return err
-		}
-
 		err = netlink.LinkSetNsFd(containerVeth, int(containerNs.Fd()))
 		if err != nil {
 			return fmt.Errorf("failed to move veth to container namespace: %s", err)
@@ -166,8 +161,6 @@ func createVeth(hostNamespace string, hostVethIfName string, containerNamespace
 	})
 	Expect(err).NotTo(HaveOccurred())
 
-	containerNs, err := ns.GetNS(containerNamespace)
-	Expect(err).NotTo(HaveOccurred())
 	err = containerNs.Do(func(_ ns.NetNS) error {
 		peerAddr := &net.IPNet{
 			IP:   hostIP,
@@ -200,7 +193,7 @@ func createVeth(hostNamespace string, hostVethIfName string, containerNamespace
 	Expect(err).NotTo(HaveOccurred())
 }
 
-func createVethInOneNs(namespace, vethName, peerName string) {
+func createVethInOneNs(netNS ns.NetNS, vethName, peerName string) {
 	vethDeviceRequest := &netlink.Veth{
 		LinkAttrs: netlink.LinkAttrs{
 			Name:  vethName,
@@ -209,10 +202,7 @@ func createVethInOneNs(namespace, vethName, peerName string) {
 		PeerName: peerName,
 	}
 
-	netNS, err := ns.GetNS(namespace)
-	Expect(err).NotTo(HaveOccurred())
-
-	err = netNS.Do(func(_ ns.NetNS) error {
+	err := netNS.Do(func(_ ns.NetNS) error {
 		if err := netlink.LinkAdd(vethDeviceRequest); err != nil {
 			return fmt.Errorf("failed to create veth pair: %v", err)
 		}
@@ -226,11 +216,8 @@ func createVethInOneNs(namespace, vethName, peerName string) {
 	Expect(err).NotTo(HaveOccurred())
 }
 
-func createMacvlan(namespace, master, macvlanName string) {
-	netNS, err := ns.GetNS(namespace)
-	Expect(err).NotTo(HaveOccurred())
-
-	err = netNS.Do(func(_ ns.NetNS) error {
+func createMacvlan(netNS ns.NetNS, master, macvlanName string) {
+	err := netNS.Do(func(_ ns.NetNS) error {
 		m, err := netlink.LinkByName(master)
 		if err != nil {
 			return fmt.Errorf("failed to lookup master %q: %v", master, err)

plugins/meta/bandwidth/main.go

@@ -23,7 +23,7 @@ import (
 
 	"github.com/containernetworking/cni/pkg/skel"
 	"github.com/containernetworking/cni/pkg/types"
-	"github.com/containernetworking/cni/pkg/types/current"
+	current "github.com/containernetworking/cni/pkg/types/100"
 	"github.com/containernetworking/cni/pkg/version"
 
 	"github.com/containernetworking/plugins/pkg/ip"
@@ -246,7 +246,7 @@ func cmdDel(args *skel.CmdArgs) error {
 }
 
 func main() {
-	skel.PluginMain(cmdAdd, cmdCheck, cmdDel, version.PluginSupports("0.3.0", "0.3.1", version.Current()), bv.BuildString("bandwidth"))
+	skel.PluginMain(cmdAdd, cmdCheck, cmdDel, version.VersionsStartingFrom("0.3.0"), bv.BuildString("bandwidth"))
 }
 
 func SafeQdiscList(link netlink.Link) ([]netlink.Qdisc, error) {

plugins/meta/firewall/README.md

@@ -1,149 +1,5 @@
-# firewall plugin
 
-## Overview
+This document has moved to the [containernetworking/cni.dev](https://github.com/containernetworking/cni.dev) repo.
 
-This plugin creates firewall rules to allow traffic to/from container IP address via the host network .
-It does not create any network interfaces and therefore does not set up connectivity by itself.
-It is intended to be used as a chained plugins.
+You can find it online here: https://cni.dev/plugins/current/meta/firewall/
 
-## Operation
-The following network configuration file
-
-```json
-{
-    "cniVersion": "0.3.1",
-    "name": "bridge-firewalld",
-    "plugins": [
-      {
-        "type": "bridge",
-        "bridge": "cni0",
-        "isGateway": true,
-        "ipMasq": true,
-        "ipam": {
-            "type": "host-local",
-            "subnet": "10.88.0.0/16",
-            "routes": [
-                { "dst": "0.0.0.0/0" }
-            ]
-        }
-      },
-      {
-        "type": "firewall"
-      }
-    ]
-}
-```
-
-will allow any IP addresses configured by earlier plugins to send/receive traffic via the host.
-
-A successful result would simply be an empty result, unless a previous plugin passed a previous result, in which case this plugin will return that previous result.
-
-## Backends
-
-This plugin supports multiple firewall backends that implement the desired functionality.
-Available backends include `iptables` and `firewalld` and may be selected with the `backend` key.
-If no `backend` key is given, the plugin will use firewalld if the service exists on the D-Bus system bus.
-If no firewalld service is found, it will fall back to iptables.
-
-## firewalld backend rule structure
-When the `firewalld` backend is used, this example will place the IPAM allocated address for the container (e.g. 10.88.0.2) into firewalld's `trusted` zone, allowing it to send/receive traffic.
-
-
-A sample standalone config list (with the file extension .conflist) using firewalld backend might
-look like:
-
-```json
-{
-    "cniVersion": "0.3.1",
-    "name": "bridge-firewalld",
-    "plugins": [
-      {
-        "type": "bridge",
-        "bridge": "cni0",
-        "isGateway": true,
-        "ipMasq": true,
-        "ipam": {
-            "type": "host-local",
-            "subnet": "10.88.0.0/16",
-            "routes": [
-                { "dst": "0.0.0.0/0" }
-            ]
-        }
-      },
-      {
-        "type": "firewall",
-        "backend": "firewalld"
-      }
-    ]
-}
-```
-
-
-`FORWARD_IN_ZONES_SOURCE` chain:
-- `-d 10.88.0.2 -j FWDI_trusted`
-
-`CNI_FORWARD_OUT_ZONES_SOURCE` chain:
-- `-s 10.88.0.2 -j FWDO_trusted`
-
-
-## iptables backend rule structure
-
-A sample standalone config list (with the file extension .conflist) using iptables backend might
-look like:
-
-```json
-{
-    "cniVersion": "0.3.1",
-    "name": "bridge-firewalld",
-    "plugins": [
-      {
-        "type": "bridge",
-        "bridge": "cni0",
-        "isGateway": true,
-        "ipMasq": true,
-        "ipam": {
-            "type": "host-local",
-            "subnet": "10.88.0.0/16",
-            "routes": [
-                { "dst": "0.0.0.0/0" }
-            ]
-        }
-      },
-      {
-        "type": "firewall",
-        "backend": "iptables"
-      }
-    ]
-}
-```
-
-When the `iptables` backend is used, the above example will create two new iptables chains in the `filter` table and add rules that allow the given interface to send/receive traffic.
-
-### FORWARD
-A new chain, CNI-FORWARD is added to the FORWARD chain.  CNI-FORWARD is the chain where rules will be added
-when containers are created and from where rules will be removed when containers terminate.
-
-`FORWARD` chain:
-- `-j CNI-FORWARD`
-
-CNI-FORWARD will have a pair of rules added, one for each direction, using the IPAM assigned IP address
-of the container as shown:
-
-`CNI-FORWARD` chain:
-- `-s 10.88.0.2 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT`
-- `-d 10.88.0.2 -j ACCEPT`
-
-The `CNI-FORWARD` chain first sends all traffic to `CNI-ADMIN` chain, which is intended as an user-controlled chain for custom rules that run prior to rules managed by the `firewall` plugin. The `firewall` plugin does not add, delete or modify rules in the `CNI-ADMIN` chain.
-
-`CNI-FORWARD` chain:
-- `-j CNI-ADMIN`
-
-The chain name `CNI-ADMIN` can be overridden by specifying `iptablesAdminChainName` in the plugin configuration
-
-```
-      {
-        "type": "firewall",
-	"backend": "iptables",
-	"iptablesAdminChainName": "SOME-OTHER-CHAIN-NAME",
-      }
-```

plugins/meta/firewall/firewall.go

@@ -24,7 +24,7 @@ import (
 
 	"github.com/containernetworking/cni/pkg/skel"
 	"github.com/containernetworking/cni/pkg/types"
-	"github.com/containernetworking/cni/pkg/types/current"
+	current "github.com/containernetworking/cni/pkg/types/100"
 	"github.com/containernetworking/cni/pkg/version"
 
 	bv "github.com/containernetworking/plugins/pkg/utils/buildversion"
@@ -46,8 +46,27 @@ type FirewallNetConf struct {
 	// the firewalld backend is used but the zone is not given, it defaults
 	// to 'trusted'
 	FirewalldZone string `json:"firewalldZone,omitempty"`
+
+	// IngressPolicy is an optional ingress policy.
+	// Defaults to "open".
+	IngressPolicy IngressPolicy `json:"ingressPolicy,omitempty"`
 }
 
+// IngressPolicy is an ingress policy string.
+type IngressPolicy = string
+
+const (
+	// IngressPolicyOpen ("open"): all inbound connections to the container are accepted.
+	// IngressPolicyOpen is the default ingress policy.
+	IngressPolicyOpen IngressPolicy = "open"
+
+	// IngressPolicySameBridge ("same-bridge"): connections from the same bridge are accepted, others are blocked.
+	// This is similar to how Docker libnetwork works.
+	// IngressPolicySameBridge executes `iptables` regardless to the value of `Backend`.
+	// IngressPolicySameBridge may not work as expected for non-bridge networks.
+	IngressPolicySameBridge IngressPolicy = "same-bridge"
+)
+
 type FirewallBackend interface {
 	Add(*FirewallNetConf, *current.Result) error
 	Del(*FirewallNetConf, *current.Result) error
@@ -129,8 +148,14 @@ func cmdAdd(args *skel.CmdArgs) error {
 		return err
 	}
 
+	if err := setupIngressPolicy(conf, result); err != nil {
+		return err
+	}
+
 	if result == nil {
-		result = &current.Result{}
+		result = &current.Result{
+			CNIVersion: current.ImplementedSpecVersion,
+		}
 	}
 	return types.PrintResult(result, conf.CNIVersion)
 }
@@ -151,11 +176,15 @@ func cmdDel(args *skel.CmdArgs) error {
 		return err
 	}
 
+	if err := teardownIngressPolicy(conf, result); err != nil {
+		return err
+	}
+
 	return nil
 }
 
 func main() {
-	skel.PluginMain(cmdAdd, cmdCheck, cmdDel, version.PluginSupports("0.4.0"), bv.BuildString("firewall"))
+	skel.PluginMain(cmdAdd, cmdCheck, cmdDel, version.VersionsStartingFrom("0.4.0"), bv.BuildString("firewall"))
 }
 
 func cmdCheck(args *skel.CmdArgs) error {

plugins/meta/firewall/firewall_firewalld_test.go

@@ -24,40 +24,17 @@ import (
 
 	"github.com/containernetworking/cni/pkg/invoke"
 	"github.com/containernetworking/cni/pkg/skel"
-	"github.com/containernetworking/cni/pkg/types/current"
+	current "github.com/containernetworking/cni/pkg/types/100"
 	"github.com/containernetworking/plugins/pkg/ns"
 	"github.com/containernetworking/plugins/pkg/testutils"
 
-	"github.com/godbus/dbus"
+	"github.com/godbus/dbus/v5"
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 )
 
-const (
-	confTmpl = `{
-  "cniVersion": "0.3.1",
-  "name": "firewalld-test",
-  "type": "firewall",
-  "backend": "firewalld",
-  "zone": "trusted",
-  "prevResult": {
-    "cniVersion": "0.3.0",
-    "interfaces": [
-      {"name": "%s", "sandbox": "%s"}
-    ],
-    "ips": [
-      {
-        "version": "4",
-        "address": "10.0.0.2/24",
-        "gateway": "10.0.0.1",
-        "interface": 0
-      }
-    ]
-  }
-}`
-	ifname = "eth0"
-)
+const ifname = "eth0"
 
 type fakeFirewalld struct {
 	zone   string
@@ -125,6 +102,30 @@ func spawnSessionDbus(wg *sync.WaitGroup) (string, *exec.Cmd) {
 	return busAddr, cmd
 }
 
+func makeFirewalldConf(ver, ifname string, ns ns.NetNS) []byte {
+	return []byte(fmt.Sprintf(`{
+	  "cniVersion": "%s",
+	  "name": "firewalld-test",
+	  "type": "firewall",
+	  "backend": "firewalld",
+	  "zone": "trusted",
+	  "prevResult": {
+	    "cniVersion": "%s",
+	    "interfaces": [
+	      {"name": "%s", "sandbox": "%s"}
+	    ],
+	    "ips": [
+	      {
+		"version": "4",
+		"address": "10.0.0.2/24",
+		"gateway": "10.0.0.1",
+		"interface": 0
+	      }
+	    ]
+	  }
+	}`, ver, ver, ifname, ns.Path()))
+}
+
 var _ = Describe("firewalld test", func() {
 	var (
 		targetNs ns.NetNS
@@ -177,167 +178,119 @@ var _ = Describe("firewalld test", func() {
 		Expect(err).NotTo(HaveOccurred())
 
 		wg.Wait()
+
+		Expect(targetNs.Close()).To(Succeed())
+		Expect(testutils.UnmountNS(targetNs)).To(Succeed())
 	})
 
-	It("works with a 0.3.1 config", func() {
-		Expect(isFirewalldRunning()).To(BeTrue())
+	// firewall plugin requires a prevResult and thus only supports 0.3.0
+	// and later CNI versions
+	for _, ver := range []string{"0.3.0", "0.3.1", "0.4.0", "1.0.0"} {
+		// Redefine ver inside for scope so real value is picked up by each dynamically defined It()
+		// See Gingkgo's "Patterns for dynamically generating tests" documentation.
+		ver := ver
 
-		conf := fmt.Sprintf(confTmpl, ifname, targetNs.Path())
-		args := &skel.CmdArgs{
-			ContainerID: "dummy",
-			Netns:       targetNs.Path(),
-			IfName:      ifname,
-			StdinData:   []byte(conf),
-		}
-		_, _, err := testutils.CmdAdd(targetNs.Path(), args.ContainerID, ifname, []byte(conf), func() error {
-			return cmdAdd(args)
+		It(fmt.Sprintf("[%s] works with a config", ver), func() {
+			Expect(isFirewalldRunning()).To(BeTrue())
+
+			conf := makeFirewalldConf(ver, ifname, targetNs)
+			args := &skel.CmdArgs{
+				ContainerID: "dummy",
+				Netns:       targetNs.Path(),
+				IfName:      ifname,
+				StdinData:   []byte(conf),
+			}
+			_, _, err := testutils.CmdAdd(targetNs.Path(), args.ContainerID, ifname, []byte(conf), func() error {
+				return cmdAdd(args)
+			})
+			Expect(err).NotTo(HaveOccurred())
+			Expect(fwd.zone).To(Equal("trusted"))
+			Expect(fwd.source).To(Equal("10.0.0.2/32"))
+			fwd.clear()
+
+			err = testutils.CmdDel(targetNs.Path(), args.ContainerID, ifname, func() error {
+				return cmdDel(args)
+			})
+			Expect(err).NotTo(HaveOccurred())
+			Expect(fwd.zone).To(Equal("trusted"))
+			Expect(fwd.source).To(Equal("10.0.0.2/32"))
 		})
-		Expect(err).NotTo(HaveOccurred())
-		Expect(fwd.zone).To(Equal("trusted"))
-		Expect(fwd.source).To(Equal("10.0.0.2/32"))
-		fwd.clear()
 
-		err = testutils.CmdDel(targetNs.Path(), args.ContainerID, ifname, func() error {
-			return cmdDel(args)
+		It(fmt.Sprintf("[%s] defaults to the firewalld backend", ver), func() {
+			Expect(isFirewalldRunning()).To(BeTrue())
+
+			conf := makeFirewalldConf(ver, ifname, targetNs)
+			args := &skel.CmdArgs{
+				ContainerID: "dummy",
+				Netns:       targetNs.Path(),
+				IfName:      ifname,
+				StdinData:   []byte(conf),
+			}
+			_, _, err := testutils.CmdAdd(targetNs.Path(), args.ContainerID, ifname, []byte(conf), func() error {
+				return cmdAdd(args)
+			})
+			Expect(err).NotTo(HaveOccurred())
+			Expect(fwd.zone).To(Equal("trusted"))
+			Expect(fwd.source).To(Equal("10.0.0.2/32"))
 		})
-		Expect(err).NotTo(HaveOccurred())
-		Expect(fwd.zone).To(Equal("trusted"))
-		Expect(fwd.source).To(Equal("10.0.0.2/32"))
-	})
 
-	It("defaults to the firewalld backend", func() {
-		conf := `{
-		  "cniVersion": "0.3.1",
-		  "name": "firewalld-test",
-		  "type": "firewall",
-		  "zone": "trusted",
-		  "prevResult": {
-		    "cniVersion": "0.3.0",
-		    "interfaces": [
-		      {"name": "eth0", "sandbox": "/foobar"}
-		    ],
-		    "ips": [
-		      {
-			"version": "4",
-			"address": "10.0.0.2/24",
-			"gateway": "10.0.0.1",
-			"interface": 0
-		      }
-		    ]
-		  }
-		}`
+		It(fmt.Sprintf("[%s] passes through the prevResult", ver), func() {
+			Expect(isFirewalldRunning()).To(BeTrue())
 
-		Expect(isFirewalldRunning()).To(BeTrue())
+			conf := makeFirewalldConf(ver, ifname, targetNs)
+			args := &skel.CmdArgs{
+				ContainerID: "dummy",
+				Netns:       targetNs.Path(),
+				IfName:      ifname,
+				StdinData:   []byte(conf),
+			}
+			r, _, err := testutils.CmdAdd(targetNs.Path(), args.ContainerID, ifname, []byte(conf), func() error {
+				return cmdAdd(args)
+			})
+			Expect(err).NotTo(HaveOccurred())
 
-		args := &skel.CmdArgs{
-			ContainerID: "dummy",
-			Netns:       targetNs.Path(),
-			IfName:      ifname,
-			StdinData:   []byte(conf),
-		}
-		_, _, err := testutils.CmdAdd(targetNs.Path(), args.ContainerID, ifname, []byte(conf), func() error {
-			return cmdAdd(args)
+			result, err := current.GetResult(r)
+			Expect(err).NotTo(HaveOccurred())
+
+			Expect(len(result.Interfaces)).To(Equal(1))
+			Expect(result.Interfaces[0].Name).To(Equal("eth0"))
+			Expect(len(result.IPs)).To(Equal(1))
+			Expect(result.IPs[0].Address.String()).To(Equal("10.0.0.2/24"))
 		})
-		Expect(err).NotTo(HaveOccurred())
-		Expect(fwd.zone).To(Equal("trusted"))
-		Expect(fwd.source).To(Equal("10.0.0.2/32"))
-	})
 
-	It("passes through the prevResult", func() {
-		conf := `{
-		  "cniVersion": "0.3.1",
-		  "name": "firewalld-test",
-		  "type": "firewall",
-		  "zone": "trusted",
-		  "prevResult": {
-		    "cniVersion": "0.3.0",
-		    "interfaces": [
-		      {"name": "eth0", "sandbox": "/foobar"}
-		    ],
-		    "ips": [
-		      {
-			"version": "4",
-			"address": "10.0.0.2/24",
-			"gateway": "10.0.0.1",
-			"interface": 0
-		      }
-		    ]
-		  }
-		}`
+		It(fmt.Sprintf("[%s] works with Check", ver), func() {
+			Expect(isFirewalldRunning()).To(BeTrue())
 
-		Expect(isFirewalldRunning()).To(BeTrue())
+			conf := makeFirewalldConf(ver, ifname, targetNs)
+			args := &skel.CmdArgs{
+				ContainerID: "dummy",
+				Netns:       targetNs.Path(),
+				IfName:      ifname,
+				StdinData:   []byte(conf),
+			}
+			r, _, err := testutils.CmdAddWithArgs(args, func() error {
+				return cmdAdd(args)
+			})
+			Expect(err).NotTo(HaveOccurred())
+			Expect(fwd.zone).To(Equal("trusted"))
+			Expect(fwd.source).To(Equal("10.0.0.2/32"))
 
-		args := &skel.CmdArgs{
-			ContainerID: "dummy",
-			Netns:       targetNs.Path(),
-			IfName:      ifname,
-			StdinData:   []byte(conf),
-		}
-		r, _, err := testutils.CmdAdd(targetNs.Path(), args.ContainerID, ifname, []byte(conf), func() error {
-			return cmdAdd(args)
+			if testutils.SpecVersionHasCHECK(ver) {
+				_, err = current.GetResult(r)
+				Expect(err).NotTo(HaveOccurred())
+
+				err = testutils.CmdCheckWithArgs(args, func() error {
+					return cmdCheck(args)
+				})
+				Expect(err).NotTo(HaveOccurred())
+			}
+
+			err = testutils.CmdDelWithArgs(args, func() error {
+				return cmdDel(args)
+			})
+			Expect(err).NotTo(HaveOccurred())
+			Expect(fwd.zone).To(Equal("trusted"))
+			Expect(fwd.source).To(Equal("10.0.0.2/32"))
 		})
-		Expect(err).NotTo(HaveOccurred())
-
-		result, err := current.GetResult(r)
-		Expect(err).NotTo(HaveOccurred())
-
-		Expect(len(result.Interfaces)).To(Equal(1))
-		Expect(result.Interfaces[0].Name).To(Equal("eth0"))
-		Expect(len(result.IPs)).To(Equal(1))
-		Expect(result.IPs[0].Address.String()).To(Equal("10.0.0.2/24"))
-	})
-
-	It("works with a 0.4.0 config, including Check", func() {
-		Expect(isFirewalldRunning()).To(BeTrue())
-
-		conf := `{
-			  "cniVersion": "0.4.0",
-			  "name": "firewalld-test",
-			  "type": "firewall",
-			  "backend": "firewalld",
-			  "zone": "trusted",
-			  "prevResult": {
-			    "cniVersion": "0.4.0",
-			    "interfaces": [
-			      {"name": "eth0", "sandbox": "/foobar"}
-			    ],
-			    "ips": [
-			      {
-				"version": "4",
-				"address": "10.0.0.2/24",
-				"gateway": "10.0.0.1",
-				"interface": 0
-			      }
-			    ]
-			  }
-			}`
-
-		args := &skel.CmdArgs{
-			ContainerID: "dummy",
-			Netns:       targetNs.Path(),
-			IfName:      ifname,
-			StdinData:   []byte(conf),
-		}
-		r, _, err := testutils.CmdAddWithArgs(args, func() error {
-			return cmdAdd(args)
-		})
-		Expect(err).NotTo(HaveOccurred())
-		Expect(fwd.zone).To(Equal("trusted"))
-		Expect(fwd.source).To(Equal("10.0.0.2/32"))
-
-		_, err = current.GetResult(r)
-		Expect(err).NotTo(HaveOccurred())
-
-		err = testutils.CmdCheckWithArgs(args, func() error {
-			return cmdCheck(args)
-		})
-		Expect(err).NotTo(HaveOccurred())
-
-		err = testutils.CmdDelWithArgs(args, func() error {
-			return cmdDel(args)
-		})
-		Expect(err).NotTo(HaveOccurred())
-		Expect(fwd.zone).To(Equal("trusted"))
-		Expect(fwd.source).To(Equal("10.0.0.2/32"))
-	})
+	}
 })

plugins/meta/firewall/firewall_integ_test.go

@@ -0,0 +1,203 @@
+// Copyright 2022 CNI authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+
+	"github.com/containernetworking/cni/libcni"
+	types100 "github.com/containernetworking/cni/pkg/types/100"
+	"github.com/containernetworking/plugins/pkg/ns"
+	"github.com/containernetworking/plugins/pkg/testutils"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+// The integration tests expect the "firewall" binary to be present in $PATH.
+// To run test, e.g, : go test -exec "sudo -E PATH=$(pwd):/opt/cni/bin:$PATH" -v -ginkgo.v
+var _ = Describe("firewall integration tests (ingressPolicy: same-bridge)", func() {
+	// ns0: foo (10.88.3.0/24)
+	// ns1: foo (10.88.3.0/24)
+	// ns2: bar (10.88.4.0/24)
+	//
+	// ns0@foo can talk to ns1@foo, but cannot talk to ns2@bar
+	const nsCount = 3
+	var (
+		configListFoo *libcni.NetworkConfigList // "foo", 10.88.3.0/24
+		configListBar *libcni.NetworkConfigList // "bar", 10.88.4.0/24
+		cniConf       *libcni.CNIConfig
+		namespaces    [nsCount]ns.NetNS
+	)
+
+	BeforeEach(func() {
+		var err error
+		rawConfigFoo := `
+{
+   "cniVersion": "1.0.0",
+   "name": "foo",
+   "plugins": [
+      {
+         "type": "bridge",
+         "bridge": "foo",
+         "isGateway": true,
+         "ipMasq": true,
+         "hairpinMode": true,
+         "ipam": {
+            "type": "host-local",
+            "routes": [
+               {
+                  "dst": "0.0.0.0/0"
+               }
+            ],
+            "ranges": [
+               [
+                  {
+                     "subnet": "10.88.3.0/24",
+                     "gateway": "10.88.3.1"
+                  }
+               ]
+            ]
+         }
+      },
+      {
+         "type": "firewall",
+         "backend": "iptables",
+         "ingressPolicy": "same-bridge"
+      }
+   ]
+}
+`
+		configListFoo, err = libcni.ConfListFromBytes([]byte(rawConfigFoo))
+		Expect(err).NotTo(HaveOccurred())
+
+		rawConfigBar := strings.ReplaceAll(rawConfigFoo, "foo", "bar")
+		rawConfigBar = strings.ReplaceAll(rawConfigBar, "10.88.3.", "10.88.4.")
+
+		configListBar, err = libcni.ConfListFromBytes([]byte(rawConfigBar))
+		Expect(err).NotTo(HaveOccurred())
+
+		// turn PATH in to CNI_PATH.
+		_, err = exec.LookPath("firewall")
+		Expect(err).NotTo(HaveOccurred())
+		dirs := filepath.SplitList(os.Getenv("PATH"))
+		cniConf = &libcni.CNIConfig{Path: dirs}
+
+		for i := 0; i < nsCount; i++ {
+			targetNS, err := testutils.NewNS()
+			Expect(err).NotTo(HaveOccurred())
+			fmt.Fprintf(GinkgoWriter, "namespace %d:%s\n", i, targetNS.Path())
+			namespaces[i] = targetNS
+		}
+	})
+
+	AfterEach(func() {
+		for _, targetNS := range namespaces {
+			if targetNS != nil {
+				targetNS.Close()
+			}
+		}
+	})
+
+	Describe("Testing with network foo and bar", func() {
+		It("should isolate foo from bar", func() {
+			var results [nsCount]*types100.Result
+			for i := 0; i < nsCount; i++ {
+				runtimeConfig := libcni.RuntimeConf{
+					ContainerID: fmt.Sprintf("test-cni-firewall-%d", i),
+					NetNS:       namespaces[i].Path(),
+					IfName:      "eth0",
+				}
+
+				configList := configListFoo
+				switch i {
+				case 0, 1:
+				// leave foo
+				default:
+					configList = configListBar
+				}
+
+				// Clean up garbages produced during past failed executions
+				_ = cniConf.DelNetworkList(context.TODO(), configList, &runtimeConfig)
+
+				// Make delete idempotent, so we can clean up on failure
+				netDeleted := false
+				deleteNetwork := func() error {
+					if netDeleted {
+						return nil
+					}
+					netDeleted = true
+					return cniConf.DelNetworkList(context.TODO(), configList, &runtimeConfig)
+				}
+				// Create the network
+				res, err := cniConf.AddNetworkList(context.TODO(), configList, &runtimeConfig)
+				Expect(err).NotTo(HaveOccurred())
+				// nolint: errcheck
+				defer deleteNetwork()
+
+				results[i], err = types100.NewResultFromResult(res)
+				Expect(err).NotTo(HaveOccurred())
+				fmt.Fprintf(GinkgoWriter, "results[%d]: %+v\n", i, results[i])
+			}
+			ping := func(src, dst int) error {
+				return namespaces[src].Do(func(ns.NetNS) error {
+					defer GinkgoRecover()
+					saddr := results[src].IPs[0].Address.IP.String()
+					daddr := results[dst].IPs[0].Address.IP.String()
+					srcNetName := results[src].Interfaces[0].Name
+					dstNetName := results[dst].Interfaces[0].Name
+
+					fmt.Fprintf(GinkgoWriter, "ping %s (ns%d@%s) -> %s (ns%d@%s)...",
+						saddr, src, srcNetName, daddr, dst, dstNetName)
+					timeoutSec := 1
+					if err := testutils.Ping(saddr, daddr, timeoutSec); err != nil {
+						fmt.Fprintln(GinkgoWriter, "unpingable")
+						return err
+					}
+					fmt.Fprintln(GinkgoWriter, "pingable")
+					return nil
+				})
+			}
+
+			// ns0@foo can ping to ns1@foo
+			err := ping(0, 1)
+			Expect(err).NotTo(HaveOccurred())
+
+			// ns1@foo can ping to ns0@foo
+			err = ping(1, 0)
+			Expect(err).NotTo(HaveOccurred())
+
+			// ns0@foo cannot ping to ns2@bar
+			err = ping(0, 2)
+			Expect(err).To(HaveOccurred())
+
+			// ns1@foo cannot ping to ns2@bar
+			err = ping(1, 2)
+			Expect(err).To(HaveOccurred())
+
+			// ns2@bar cannot ping to ns0@foo
+			err = ping(2, 0)
+			Expect(err).To(HaveOccurred())
+
+			// ns2@bar cannot ping to ns1@foo
+			err = ping(2, 1)
+			Expect(err).To(HaveOccurred())
+		})
+	})
+})