refactor(win-bridge): netconf

- support v2 api
- unify v1 and v2 api

BREAKING CHANGE:
- remove `HcnPolicyArgs` field
- merge `HcnPolicyArgs` into `Policies` field

Signed-off-by: thxcode <thxcode0824@gmail.com>

pkg/hns/endpoint_windows.go

@@ -53,7 +53,7 @@ func GetSandboxContainerID(containerID string, netNs string) string {
 	return containerID
 }
 
-// GetIpString returns the given IP in string.
+// GetIpString returns the given IP as a string.
 func GetIpString(ip *net.IP) string {
 	if len(*ip) == 0 {
 		return ""
@@ -65,7 +65,7 @@ func GetIpString(ip *net.IP) string {
 // GetDefaultDestinationPrefix returns the default destination prefix according to the given IP type.
 func GetDefaultDestinationPrefix(ip *net.IP) string {
 	destinationPrefix := "0.0.0.0/0"
-	if ipv6 := ip.To4(); ipv6 == nil {
+	if ip.To4() == nil {
 		destinationPrefix = "::/0"
 	}
 	return destinationPrefix
@@ -95,7 +95,7 @@ func GenerateHnsEndpoint(epInfo *EndpointInfo, n *NetConf) (*hcsshim.HNSEndpoint
 	}
 
 	if n.LoopbackDSR {
-		n.ApplyLoopbackDSR(&epInfo.IpAddress)
+		n.ApplyLoopbackDSRPolicy(&epInfo.IpAddress)
 	}
 	hnsEndpoint = &hcsshim.HNSEndpoint{
 		Name:           epInfo.EndpointName,
@@ -104,7 +104,7 @@ func GenerateHnsEndpoint(epInfo *EndpointInfo, n *NetConf) (*hcsshim.HNSEndpoint
 		DNSSuffix:      strings.Join(epInfo.DNS.Search, ","),
 		GatewayAddress: GetIpString(&epInfo.Gateway),
 		IPAddress:      epInfo.IpAddress,
-		Policies:       n.MarshalPolicies(),
+		Policies:       n.GetHNSEndpointPolicies(),
 	}
 	return hnsEndpoint, nil
 }
@@ -240,7 +240,7 @@ func GenerateHcnEndpoint(epInfo *EndpointInfo, n *NetConf) (*hcn.HostComputeEndp
 	}
 
 	if n.LoopbackDSR {
-		n.ApplyLoopbackDSR(&epInfo.IpAddress)
+		n.ApplyLoopbackDSRPolicy(&epInfo.IpAddress)
 	}
 	hcnEndpoint = &hcn.HostComputeEndpoint{
 		SchemaVersion: hcn.SchemaVersion{
@@ -266,12 +266,7 @@ func GenerateHcnEndpoint(epInfo *EndpointInfo, n *NetConf) (*hcn.HostComputeEndp
 				IpAddress: GetIpString(&epInfo.IpAddress),
 			},
 		},
-		Policies: func() []hcn.EndpointPolicy {
+		Policies: n.GetHostComputeEndpointPolicies(),
-			if n.HcnPolicyArgs == nil {
-				n.HcnPolicyArgs = []hcn.EndpointPolicy{}
-			}
-			return n.HcnPolicyArgs
-		}(),
 	}
 	return hcnEndpoint, nil
 }

pkg/hns/netconf_suite_windows_test.go

@@ -20,7 +20,7 @@ import (
 	"testing"
 )
 
-func TestHns(t *testing.T) {
+func TestNetConf(t *testing.T) {
 	RegisterFailHandler(Fail)
-	RunSpecs(t, "HNS NetConf Suite")
+	RunSpecs(t, "NetConf Suite")
 }

pkg/hns/netconf_windows.go

@@ -17,8 +17,10 @@ package hns
 import (
 	"bytes"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"net"
+	"strconv"
 	"strings"
 
 	"github.com/Microsoft/hcsshim/hcn"
@@ -29,16 +31,16 @@ import (
 // NetConf is the CNI spec
 type NetConf struct {
 	types.NetConf
-	// ApiVersion is either 1 or 2, which specifies which hns APIs to call
+	// ApiVersion specifies the policies type of HNS or HCN, select one of [1, 2].
-	ApiVersion int `json:"ApiVersion"`
+	// HNS is the v1 API, which is the default version and applies to dockershim.
-	// V2 Api Policies
+	// HCN is the v2 API, which can leverage HostComputeNamespace and use in containerd.
-	HcnPolicyArgs []hcn.EndpointPolicy `json:"HcnPolicyArgs,omitempty"`
+	ApiVersion int `json:"apiVersion,omitempty"`
-	// V1 Api Policies
+	// Policies specifies the policy list for HNSEndpoint or HostComputeEndpoint.
-	Policies []policy `json:"policies,omitempty"`
+	Policies []Policy `json:"policies,omitempty"`
-	// Options to be passed in by the runtime
+	// RuntimeConfig represents the options to be passed in by the runtime.
 	RuntimeConfig RuntimeConfig `json:"runtimeConfig"`
-	// If true, adds a policy to endpoints to support loopback direct server return
+	// LoopbackDSR specifies whether to support loopback direct server return.
-	LoopbackDSR bool `json:"loopbackDSR"`
+	LoopbackDSR bool `json:"loopbackDSR,omitempty"`
 }
 
 type RuntimeDNS struct {
@@ -53,32 +55,63 @@ type PortMapEntry struct {
 	HostIP        string `json:"hostIP,omitempty"`
 }
 
+// constants of the supported Windows Socket protocol,
+// ref to https://docs.microsoft.com/en-us/dotnet/api/system.net.sockets.protocoltype.
+var protocolEnums = map[string]uint32{
+	"icmpv4": 1,
+	"igmp":   2,
+	"tcp":    6,
+	"udp":    17,
+	"icmpv6": 58,
+}
+
+func (p *PortMapEntry) GetProtocolEnum() (uint32, error) {
+	var u, err = strconv.ParseUint(p.Protocol, 0, 10)
+	if err != nil {
+		var pe, exist = protocolEnums[strings.ToLower(p.Protocol)]
+		if !exist {
+			return 0, errors.New("invalid protocol supplied to port mapping policy")
+		}
+		return pe, nil
+	}
+	return uint32(u), nil
+}
+
 type RuntimeConfig struct {
 	DNS      RuntimeDNS     `json:"dns"`
 	PortMaps []PortMapEntry `json:"portMappings,omitempty"`
 }
 
-type policy struct {
+type Policy struct {
 	Name  string          `json:"name"`
 	Value json.RawMessage `json:"value"`
 }
 
-// MarshalPolicies converts the HNSEndpoint policies in Policies
+// GetHNSEndpointPolicies converts the configuration policies to HNSEndpoint policies.
-// to HNS specific policies as Json raw bytes.
+func (n *NetConf) GetHNSEndpointPolicies() []json.RawMessage {
-func (n *NetConf) MarshalPolicies() []json.RawMessage {
-	if n.Policies == nil {
-		n.Policies = make([]policy, 0)
-	}
-
 	result := make([]json.RawMessage, 0, len(n.Policies))
 	for _, p := range n.Policies {
 		if !strings.EqualFold(p.Name, "EndpointPolicy") {
 			continue
 		}
-
 		result = append(result, p.Value)
 	}
+	return result
+}
 
+// GetHostComputeEndpointPolicies converts the configuration policies to HostComputeEndpoint policies.
+func (n *NetConf) GetHostComputeEndpointPolicies() []hcn.EndpointPolicy {
+	result := make([]hcn.EndpointPolicy, 0, len(n.Policies))
+	for _, p := range n.Policies {
+		if !strings.EqualFold(p.Name, "EndpointPolicy") {
+			continue
+		}
+		var policy hcn.EndpointPolicy
+		if err := json.Unmarshal(p.Value, &policy); err != nil {
+			continue
+		}
+		result = append(result, policy)
+	}
 	return result
 }
 
@@ -94,134 +127,222 @@ func (n *NetConf) GetDNS() types.DNS {
 	return dnsResult
 }
 
-// ApplyLoopbackDSR configures the given IP to support loopback DSR.
+// ApplyLoopbackDSRPolicy configures the given IP to support loopback DSR.
-func (n *NetConf) ApplyLoopbackDSR(ip *net.IP) {
+func (n *NetConf) ApplyLoopbackDSRPolicy(ip *net.IP) {
-	value := fmt.Sprintf(`"Destinations" : ["%s"]`, ip.String())
+	if err := hcn.DSRSupported(); err != nil || ip == nil {
+		return
+	}
+
+	toPolicyValue := func(addr string) json.RawMessage {
 		if n.ApiVersion == 2 {
-		hcnLoopbackRoute := hcn.EndpointPolicy{
+			return bprintf(`{"Type": "OutBoundNAT", "Settings": {"Destinations": ["%s"]}}`, addr)
-			Type:     "OutBoundNAT",
-			Settings: []byte(fmt.Sprintf("{%s}", value)),
-		}
-		n.HcnPolicyArgs = append(n.HcnPolicyArgs, hcnLoopbackRoute)
-	} else {
-		hnsLoopbackRoute := policy{
-			Name:  "EndpointPolicy",
-			Value: []byte(fmt.Sprintf(`{"Type": "OutBoundNAT", %s}`, value)),
-		}
-		n.Policies = append(n.Policies, hnsLoopbackRoute)
 		}
+		return bprintf(`{"Type": "OutBoundNAT", "Destinations": ["%s"]}`, addr)
 	}
+	ipBytes := []byte(ip.String())
 
-// ApplyOutboundNatPolicy applies the sNAT policy in HNS/HCN and configures the given CIDR as an exception.
+	// find OutBoundNAT policy
-func (n *NetConf) ApplyOutboundNatPolicy(nwToNat string) {
+	for i := range n.Policies {
-	if n.Policies == nil {
+		p := &n.Policies[i]
-		n.Policies = make([]policy, 0)
-	}
-
-	nwToNatBytes := []byte(nwToNat)
-
-	for i, p := range n.Policies {
 		if !strings.EqualFold(p.Name, "EndpointPolicy") {
 			continue
 		}
 
-		typeValue, err := jsonparser.GetUnsafeString(p.Value, "Type")
+		// filter OutBoundNAT policy
-		if err != nil || len(typeValue) == 0 {
+		typeValue, _ := jsonparser.GetUnsafeString(p.Value, "Type")
+		if typeValue != "OutBoundNAT" {
 			continue
 		}
 
-		if !strings.EqualFold(typeValue, "OutBoundNAT") {
+		// parse destination address list
+		var (
+			destinationsValue []byte
+			dt                jsonparser.ValueType
+		)
+		if n.ApiVersion == 2 {
+			destinationsValue, dt, _, _ = jsonparser.Get(p.Value, "Settings", "Destinations")
+		} else {
+			destinationsValue, dt, _, _ = jsonparser.Get(p.Value, "Destinations")
+		}
+
+		// skip if Destinations/DestinationList field is not found
+		if dt == jsonparser.NotExist {
 			continue
 		}
 
-		exceptionListValue, dt, _, _ := jsonparser.Get(p.Value, "ExceptionList")
+		// return if found the given address
-		// OutBoundNAT must with ExceptionList, so don't need to judge jsonparser.NotExist
 		if dt == jsonparser.Array {
-			buf := bytes.Buffer{}
+			var found bool
-			buf.WriteString(`{"Type": "OutBoundNAT", "ExceptionList": [`)
+			_, _ = jsonparser.ArrayEach(destinationsValue, func(value []byte, dataType jsonparser.ValueType, offset int, err error) {
-
-			jsonparser.ArrayEach(exceptionListValue, func(value []byte, dataType jsonparser.ValueType, offset int, err error) {
 				if dataType == jsonparser.String && len(value) != 0 {
-					if bytes.Compare(value, nwToNatBytes) != 0 {
+					if bytes.Compare(value, ipBytes) == 0 {
-						buf.WriteByte('"')
+						found = true
-						buf.Write(value)
-						buf.WriteByte('"')
-						buf.WriteByte(',')
 					}
 				}
 			})
+			if found {
+				return
+			}
+		}
+	}
 
-			buf.WriteString(`"` + nwToNat + `"]}`)
+	// or add a new OutBoundNAT if not found
-
+	n.Policies = append(n.Policies, Policy{
-			n.Policies[i] = policy{
 		Name:  "EndpointPolicy",
-				Value: buf.Bytes(),
+		Value: toPolicyValue(ip.String()),
-			}
+	})
-		} else {
-			n.Policies[i] = policy{
-				Name:  "EndpointPolicy",
-				Value: []byte(`{"Type": "OutBoundNAT", "ExceptionList": ["` + nwToNat + `"]}`),
-			}
 }
 
+// ApplyOutboundNatPolicy applies the sNAT policy in HNS/HCN and configures the given CIDR as an exception.
+func (n *NetConf) ApplyOutboundNatPolicy(exceptionCIDR string) {
+	if exceptionCIDR == "" {
 		return
 	}
 
-	// didn't find the policyArg, add it
+	toPolicyValue := func(cidr ...string) json.RawMessage {
-	n.Policies = append(n.Policies, policy{
+		if n.ApiVersion == 2 {
+			return bprintf(`{"Type": "OutBoundNAT", "Settings": {"Exceptions": ["%s"]}}`, strings.Join(cidr, `","`))
+		}
+		return bprintf(`{"Type": "OutBoundNAT", "ExceptionList": ["%s"]}`, strings.Join(cidr, `","`))
+	}
+	exceptionCIDRBytes := []byte(exceptionCIDR)
+
+	// find OutBoundNAT policy
+	for i := range n.Policies {
+		p := &n.Policies[i]
+		if !strings.EqualFold(p.Name, "EndpointPolicy") {
+			continue
+		}
+
+		// filter OutBoundNAT policy
+		typeValue, _ := jsonparser.GetUnsafeString(p.Value, "Type")
+		if typeValue != "OutBoundNAT" {
+			continue
+		}
+
+		// parse exception CIDR list
+		var (
+			exceptionsValue []byte
+			dt              jsonparser.ValueType
+		)
+		if n.ApiVersion == 2 {
+			exceptionsValue, dt, _, _ = jsonparser.Get(p.Value, "Settings", "Exceptions")
+		} else {
+			exceptionsValue, dt, _, _ = jsonparser.Get(p.Value, "ExceptionList")
+		}
+
+		// skip if Exceptions/ExceptionList field is not found
+		if dt == jsonparser.NotExist {
+			continue
+		}
+
+		// return if found the given CIDR
+		if dt == jsonparser.Array {
+			var found bool
+			_, _ = jsonparser.ArrayEach(exceptionsValue, func(value []byte, dataType jsonparser.ValueType, offset int, err error) {
+				if dataType == jsonparser.String && len(value) != 0 {
+					if bytes.Compare(value, exceptionCIDRBytes) == 0 {
+						found = true
+					}
+				}
+			})
+			if found {
+				return
+			}
+		}
+	}
+
+	// or add a new OutBoundNAT if not found
+	n.Policies = append(n.Policies, Policy{
 		Name:  "EndpointPolicy",
-		Value: []byte(`{"Type": "OutBoundNAT", "ExceptionList": ["` + nwToNat + `"]}`),
+		Value: toPolicyValue(exceptionCIDR),
 	})
 }
 
 // ApplyDefaultPAPolicy applies an endpoint PA policy in HNS/HCN.
-func (n *NetConf) ApplyDefaultPAPolicy(paAddress string) {
+func (n *NetConf) ApplyDefaultPAPolicy(address string) {
-	if n.Policies == nil {
+	if address == "" {
-		n.Policies = make([]policy, 0)
+		return
 	}
 
-	// if its already present, leave untouched
+	toPolicyValue := func(addr string) json.RawMessage {
-	for i, p := range n.Policies {
+		if n.ApiVersion == 2 {
+			return bprintf(`{"Type": "ProviderAddress", "Settings": {"ProviderAddress": "%s"}}`, addr)
+		}
+		return bprintf(`{"Type": "PA", "PA": "%s"}`, addr)
+	}
+	addressBytes := []byte(address)
+
+	// find ProviderAddress policy
+	for i := range n.Policies {
+		p := &n.Policies[i]
 		if !strings.EqualFold(p.Name, "EndpointPolicy") {
 			continue
 		}
 
-		paValue, dt, _, _ := jsonparser.Get(p.Value, "PA")
+		// filter ProviderAddress policy
+		typeValue, _ := jsonparser.GetUnsafeString(p.Value, "Type")
+		if typeValue != "PA" && typeValue != "ProviderAddress" {
+			continue
+		}
+
+		// parse provider address
+		var (
+			paValue []byte
+			dt      jsonparser.ValueType
+		)
+		if n.ApiVersion == 2 {
+			paValue, dt, _, _ = jsonparser.Get(p.Value, "Settings", "ProviderAddress")
+		} else {
+			paValue, dt, _, _ = jsonparser.Get(p.Value, "PA")
+		}
+
+		// skip if ProviderAddress/PA field is not found
 		if dt == jsonparser.NotExist {
 			continue
-		} else if dt == jsonparser.String && len(paValue) != 0 {
-			// found it, don't override
-			return
 		}
 
-		n.Policies[i] = policy{
+		// return if found the given address
-			Name:  "EndpointPolicy",
+		if dt == jsonparser.String && bytes.Compare(paValue, addressBytes) == 0 {
-			Value: []byte(`{"Type": "PA", "PA": "` + paAddress + `"}`),
-		}
 			return
 		}
+	}
 
-	// didn't find the policyArg, add it
+	// or add a new ProviderAddress if not found
-	n.Policies = append(n.Policies, policy{
+	n.Policies = append(n.Policies, Policy{
 		Name:  "EndpointPolicy",
-		Value: []byte(`{"Type": "PA", "PA": "` + paAddress + `"}`),
+		Value: toPolicyValue(address),
 	})
 }
 
 // ApplyPortMappingPolicy applies the host/container port mapping policies in HNS/HCN.
 func (n *NetConf) ApplyPortMappingPolicy(portMappings []PortMapEntry) {
-	if portMappings == nil {
+	if len(portMappings) == 0 {
 		return
 	}
 
-	if n.Policies == nil {
+	toPolicyValue := func(p *PortMapEntry) json.RawMessage {
-		n.Policies = make([]policy, 0)
+		if n.ApiVersion == 2 {
+			var protocolEnum, _ = p.GetProtocolEnum()
+			return bprintf(`{"Type": "PortMapping", "Settings": {"InternalPort": %d, "ExternalPort": %d, "Protocol": %d, "VIP": "%s"}}`, p.ContainerPort, p.HostPort, protocolEnum, p.HostIP)
+		}
+		return bprintf(`{"Type": "NAT", "InternalPort": %d, "ExternalPort": %d, "Protocol": "%s"}`, p.ContainerPort, p.HostPort, p.Protocol)
 	}
 
-	for _, portMapping := range portMappings {
+	for i := range portMappings {
-		n.Policies = append(n.Policies, policy{
+		p := &portMappings[i]
+		// skip the invalid protocol mapping
+		if _, err := p.GetProtocolEnum(); err != nil {
+			continue
+		}
+		n.Policies = append(n.Policies, Policy{
 			Name:  "EndpointPolicy",
-			Value: []byte(fmt.Sprintf(`{"Type": "NAT", "InternalPort": %d, "ExternalPort": %d, "Protocol": "%s"}`, portMapping.ContainerPort, portMapping.HostPort, portMapping.Protocol)),
+			Value: toPolicyValue(p),
 		})
 	}
 }
+
+// bprintf is similar to fmt.Sprintf and returns a byte array as result.
+func bprintf(format string, a ...interface{}) []byte {
+	return []byte(fmt.Sprintf(format, a...))
+}

pkg/hns/netconf_windows_test.go

@@ -15,221 +15,585 @@ package hns
 
 import (
 	"encoding/json"
+	"net"
 
+	"github.com/Microsoft/hcsshim/hcn"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 )
 
-var _ = Describe("HNS NetConf", func() {
+var _ = Describe("NetConf", func() {
-	Describe("ApplyOutBoundNATPolicy", func() {
+	Describe("ApplyLoopbackDSRPolicy", func() {
-		Context("when not set by user", func() {
+		Context("via v1 api", func() {
-			It("sets it by adding a policy", func() {
+			var n NetConf
+			BeforeEach(func() {
+				n = NetConf{}
+			})
 
-				// apply it
+			It("filter out duplicated IP", func() {
-				n := NetConf{}
+				// mock duplicated IP
-				n.ApplyOutboundNatPolicy("192.168.0.0/16")
+				ip := net.ParseIP("172.16.0.12")
+				n.ApplyLoopbackDSRPolicy(&ip)
+				n.ApplyLoopbackDSRPolicy(&ip)
 
+				// only one policy
 				addlArgs := n.Policies
 				Expect(addlArgs).Should(HaveLen(1))
 
+				// normal type judgement
 				policy := addlArgs[0]
 				Expect(policy.Name).Should(Equal("EndpointPolicy"))
-
 				value := make(map[string]interface{})
 				json.Unmarshal(policy.Value, &value)
-
 				Expect(value).Should(HaveKey("Type"))
-				Expect(value).Should(HaveKey("ExceptionList"))
 				Expect(value["Type"]).Should(Equal("OutBoundNAT"))
+				Expect(value).Should(HaveKey("Destinations"))
 
-				exceptionList := value["ExceptionList"].([]interface{})
+				// and only one item
-				Expect(exceptionList).Should(HaveLen(1))
+				destinationList := value["Destinations"].([]interface{})
-				Expect(exceptionList[0].(string)).Should(Equal("192.168.0.0/16"))
+				Expect(destinationList).Should(HaveLen(1))
+				Expect(destinationList[0].(string)).Should(Equal("172.16.0.12"))
+			})
+
+			It("append different IP", func() {
+				// mock different IP
+				ip1 := net.ParseIP("172.16.0.12")
+				n.ApplyLoopbackDSRPolicy(&ip1)
+				ip2 := net.ParseIP("172.16.0.13")
+				n.ApplyLoopbackDSRPolicy(&ip2)
+
+				// will be two policies
+				addlArgs := n.Policies
+				Expect(addlArgs).Should(HaveLen(2))
+
+				// normal type judgement
+				policy := addlArgs[1] // pick second item
+				Expect(policy.Name).Should(Equal("EndpointPolicy"))
+				value := make(map[string]interface{})
+				json.Unmarshal(policy.Value, &value)
+				Expect(value).Should(HaveKey("Type"))
+				Expect(value["Type"]).Should(Equal("OutBoundNAT"))
+				Expect(value).Should(HaveKey("Destinations"))
+
+				// only one item
+				destinationList := value["Destinations"].([]interface{})
+				Expect(destinationList).Should(HaveLen(1))
+				Expect(destinationList[0].(string)).Should(Equal("172.16.0.13"))
 			})
 		})
 
-		Context("when set by user", func() {
+		Context("via v2 api", func() {
-			It("appends exceptions to the existing policy", func() {
+			var n NetConf
-				// first set it
+			BeforeEach(func() {
-				n := NetConf{}
+				n = NetConf{ApiVersion: 2}
-				n.ApplyOutboundNatPolicy("192.168.0.0/16")
+			})
 
-				// then attempt to update it
+			It("filter out duplicated IP", func() {
-				n.ApplyOutboundNatPolicy("10.244.0.0/16")
+				// mock duplicated IP
+				ip := net.ParseIP("172.16.0.12")
+				n.ApplyLoopbackDSRPolicy(&ip)
+				n.ApplyLoopbackDSRPolicy(&ip)
 
-				// it should be unchanged!
+				// only one policy
 				addlArgs := n.Policies
 				Expect(addlArgs).Should(HaveLen(1))
 
+				// normal type judgement
 				policy := addlArgs[0]
 				Expect(policy.Name).Should(Equal("EndpointPolicy"))
-
+				value := make(map[string]interface{})
-				var value map[string]interface{}
 				json.Unmarshal(policy.Value, &value)
-
 				Expect(value).Should(HaveKey("Type"))
-				Expect(value).Should(HaveKey("ExceptionList"))
 				Expect(value["Type"]).Should(Equal("OutBoundNAT"))
+				Expect(value).Should(HaveKey("Settings"))
 
+				// and only one item
+				settings := value["Settings"].(map[string]interface{})
+				destinationList := settings["Destinations"].([]interface{})
+				Expect(destinationList).Should(HaveLen(1))
+				Expect(destinationList[0].(string)).Should(Equal("172.16.0.12"))
+			})
+
+			It("append different IP", func() {
+				// mock different IP
+				ip1 := net.ParseIP("172.16.0.12")
+				n.ApplyLoopbackDSRPolicy(&ip1)
+				ip2 := net.ParseIP("172.16.0.13")
+				n.ApplyLoopbackDSRPolicy(&ip2)
+
+				// will be two policies
+				addlArgs := n.Policies
+				Expect(addlArgs).Should(HaveLen(2))
+
+				// normal type judgement
+				policy := addlArgs[1] // pick second item
+				Expect(policy.Name).Should(Equal("EndpointPolicy"))
+				value := make(map[string]interface{})
+				json.Unmarshal(policy.Value, &value)
+				Expect(value).Should(HaveKey("Type"))
+				Expect(value["Type"]).Should(Equal("OutBoundNAT"))
+				Expect(value).Should(HaveKey("Settings"))
+
+				// only one item
+				settings := value["Settings"].(map[string]interface{})
+				destinationList := settings["Destinations"].([]interface{})
+				Expect(destinationList).Should(HaveLen(1))
+				Expect(destinationList[0].(string)).Should(Equal("172.16.0.13"))
+			})
+		})
+	})
+
+	Describe("ApplyOutBoundNATPolicy", func() {
+		Context("via v1 api", func() {
+			var n NetConf
+			BeforeEach(func() {
+				n = NetConf{}
+			})
+
+			It("append different IP", func() {
+				// mock different IP
+				n.ApplyOutboundNatPolicy("192.168.0.0/16")
+				n.ApplyOutboundNatPolicy("10.244.0.0/16")
+
+				// will be two policies
+				addlArgs := n.Policies
+				Expect(addlArgs).Should(HaveLen(2))
+
+				// normal type judgement
+				policy := addlArgs[1] // pick second item
+				Expect(policy.Name).Should(Equal("EndpointPolicy"))
+				value := make(map[string]interface{})
+				json.Unmarshal(policy.Value, &value)
+				Expect(value).Should(HaveKey("Type"))
+				Expect(value["Type"]).Should(Equal("OutBoundNAT"))
+				Expect(value).Should(HaveKey("ExceptionList"))
+
+				// but get two items
 				exceptionList := value["ExceptionList"].([]interface{})
-				Expect(exceptionList).Should(HaveLen(2))
+				Expect(exceptionList).Should(HaveLen(1))
-				Expect(exceptionList[0].(string)).Should(Equal("192.168.0.0/16"))
+				Expect(exceptionList[0].(string)).Should(Equal("10.244.0.0/16"))
-				Expect(exceptionList[1].(string)).Should(Equal("10.244.0.0/16"))
+			})
+
+			It("append a new one if there is not an exception OutBoundNAT policy", func() {
+				// mock different OutBoundNAT routes
+				n.Policies = []Policy{
+					{
+						Name:  "EndpointPolicy",
+						Value: bprintf(`{"Type": "OutBoundNAT", "OtherList": []}`),
+					},
+				}
+				n.ApplyOutboundNatPolicy("10.244.0.0/16")
+
+				// has two policies
+				addlArgs := n.Policies
+				Expect(addlArgs).Should(HaveLen(2))
+
+				// normal type judgement
+				policy := addlArgs[0]
+				Expect(policy.Name).Should(Equal("EndpointPolicy"))
+				value := make(map[string]interface{})
+				json.Unmarshal(policy.Value, &value)
+				Expect(value).Should(HaveKey("Type"))
+				Expect(value["Type"]).Should(Equal("OutBoundNAT"))
+				Expect(value).Should(HaveKey("OtherList"))
+				policy = addlArgs[1]
+				value = make(map[string]interface{})
+				json.Unmarshal(policy.Value, &value)
+				Expect(value).Should(HaveKey("Type"))
+				Expect(value["Type"]).Should(Equal("OutBoundNAT"))
+				Expect(value).Should(HaveKey("ExceptionList"))
+
+				// only get one item
+				exceptionList := value["ExceptionList"].([]interface{})
+				Expect(exceptionList).Should(HaveLen(1))
+				Expect(exceptionList[0].(string)).Should(Equal("10.244.0.0/16"))
+			})
+
+			It("nothing to do if CIDR is blank", func() {
+				// mock different OutBoundNAT routes
+				n.Policies = []Policy{
+					{
+						Name:  "EndpointPolicy",
+						Value: bprintf(`{"Type": "OutBoundNAT", "ExceptionList": []}`),
+					},
+				}
+				n.ApplyOutboundNatPolicy("")
+
+				// only one policy
+				addlArgs := n.Policies
+				Expect(addlArgs).Should(HaveLen(1))
+
+				// normal type judgement
+				policy := addlArgs[0]
+				Expect(policy.Name).Should(Equal("EndpointPolicy"))
+				value := make(map[string]interface{})
+				json.Unmarshal(policy.Value, &value)
+				Expect(value).Should(HaveKey("Type"))
+				Expect(value["Type"]).Should(Equal("OutBoundNAT"))
+				Expect(value).Should(HaveKey("ExceptionList"))
+
+				// empty list
+				Expect(value["ExceptionList"]).ShouldNot(BeNil())
+				Expect(value["ExceptionList"]).Should(HaveLen(0))
+			})
+		})
+
+		Context("via v2 api", func() {
+			var n NetConf
+			BeforeEach(func() {
+				n = NetConf{ApiVersion: 2}
+			})
+
+			It("append different IP", func() {
+				// mock different IP
+				n.ApplyOutboundNatPolicy("192.168.0.0/16")
+				n.ApplyOutboundNatPolicy("10.244.0.0/16")
+
+				// will be two policies
+				addlArgs := n.Policies
+				Expect(addlArgs).Should(HaveLen(2))
+
+				// normal type judgement
+				policy := addlArgs[1] // pick second item
+				Expect(policy.Name).Should(Equal("EndpointPolicy"))
+				value := make(map[string]interface{})
+				json.Unmarshal(policy.Value, &value)
+				Expect(value).Should(HaveKey("Type"))
+				Expect(value["Type"]).Should(Equal("OutBoundNAT"))
+				Expect(value).Should(HaveKey("Settings"))
+
+				// but get two items
+				settings := value["Settings"].(map[string]interface{})
+				exceptionList := settings["Exceptions"].([]interface{})
+				Expect(exceptionList).Should(HaveLen(1))
+				Expect(exceptionList[0].(string)).Should(Equal("10.244.0.0/16"))
+			})
+
+			It("append a new one if there is not an exception OutBoundNAT policy", func() {
+				// mock different OutBoundNAT routes
+				n.Policies = []Policy{
+					{
+						Name:  "EndpointPolicy",
+						Value: bprintf(`{"Type": "OutBoundNAT", "Settings": {"Others": []}}`),
+					},
+				}
+				n.ApplyOutboundNatPolicy("10.244.0.0/16")
+
+				// has two policies
+				addlArgs := n.Policies
+				Expect(addlArgs).Should(HaveLen(2))
+
+				// normal type judgement
+				policy := addlArgs[0]
+				Expect(policy.Name).Should(Equal("EndpointPolicy"))
+				value := make(map[string]interface{})
+				json.Unmarshal(policy.Value, &value)
+				Expect(value).Should(HaveKey("Type"))
+				Expect(value["Type"]).Should(Equal("OutBoundNAT"))
+				Expect(value).Should(HaveKey("Settings"))
+				Expect(value["Settings"]).Should(HaveKey("Others"))
+				policy = addlArgs[1]
+				value = make(map[string]interface{})
+				json.Unmarshal(policy.Value, &value)
+				Expect(value).Should(HaveKey("Type"))
+				Expect(value["Type"]).Should(Equal("OutBoundNAT"))
+				Expect(value).Should(HaveKey("Settings"))
+
+				// only get one item
+				settings := value["Settings"].(map[string]interface{})
+				exceptionList := settings["Exceptions"].([]interface{})
+				Expect(exceptionList).Should(HaveLen(1))
+				Expect(exceptionList[0].(string)).Should(Equal("10.244.0.0/16"))
+			})
+
+			It("nothing to do if CIDR is blank", func() {
+				// mock different OutBoundNAT routes
+				n.Policies = []Policy{
+					{
+						Name:  "EndpointPolicy",
+						Value: bprintf(`{"Type": "OutBoundNAT", "Settings": {"Exceptions": []}}`),
+					},
+				}
+				n.ApplyOutboundNatPolicy("")
+
+				// only one policy
+				addlArgs := n.Policies
+				Expect(addlArgs).Should(HaveLen(1))
+
+				// normal type judgement
+				policy := addlArgs[0]
+				Expect(policy.Name).Should(Equal("EndpointPolicy"))
+				value := make(map[string]interface{})
+				json.Unmarshal(policy.Value, &value)
+				Expect(value).Should(HaveKey("Type"))
+				Expect(value["Type"]).Should(Equal("OutBoundNAT"))
+				Expect(value).Should(HaveKey("Settings"))
+
+				// empty list
+				settings := value["Settings"].(map[string]interface{})
+				Expect(settings["Exceptions"]).ShouldNot(BeNil())
+				Expect(settings["Exceptions"]).Should(HaveLen(0))
 			})
 		})
 	})
 
 	Describe("ApplyDefaultPAPolicy", func() {
-		Context("when not set by user", func() {
+		Context("via v1 api", func() {
-			It("sets it by adding a policy", func() {
+			var n NetConf
-
+			BeforeEach(func() {
-				n := NetConf{}
+				n = NetConf{}
-				n.ApplyDefaultPAPolicy("192.168.0.1")
-
-				addlArgs := n.Policies
-				Expect(addlArgs).Should(HaveLen(1))
-
-				policy := addlArgs[0]
-				Expect(policy.Name).Should(Equal("EndpointPolicy"))
-
-				value := make(map[string]interface{})
-				json.Unmarshal(policy.Value, &value)
-
-				Expect(value).Should(HaveKey("Type"))
-				Expect(value["Type"]).Should(Equal("PA"))
-
-				paAddress := value["PA"].(string)
-				Expect(paAddress).Should(Equal("192.168.0.1"))
-			})
 			})
 
-		Context("when set by user", func() {
+			It("append different IP", func() {
-			It("does not override", func() {
+				// mock different IP
-				n := NetConf{}
 				n.ApplyDefaultPAPolicy("192.168.0.1")
 				n.ApplyDefaultPAPolicy("192.168.0.2")
 
+				// will be two policies
 				addlArgs := n.Policies
-				Expect(addlArgs).Should(HaveLen(1))
+				Expect(addlArgs).Should(HaveLen(2))
 
-				policy := addlArgs[0]
+				// normal type judgement
+				policy := addlArgs[1] // judge second item
 				Expect(policy.Name).Should(Equal("EndpointPolicy"))
-
 				value := make(map[string]interface{})
 				json.Unmarshal(policy.Value, &value)
-
 				Expect(value).Should(HaveKey("Type"))
 				Expect(value["Type"]).Should(Equal("PA"))
 
+				// compare with second item
 				paAddress := value["PA"].(string)
-				Expect(paAddress).Should(Equal("192.168.0.1"))
+				Expect(paAddress).Should(Equal("192.168.0.2"))
-				Expect(paAddress).ShouldNot(Equal("192.168.0.2"))
+			})
+
+			It("nothing to do if IP is blank", func() {
+				// mock different policy
+				n.Policies = []Policy{
+					{
+						Name:  "EndpointPolicy",
+						Value: bprintf(`{"Type": "OutBoundNAT", "Exceptions": ["192.168.0.0/16"]}`),
+					},
+				}
+				n.ApplyDefaultPAPolicy("")
+
+				// nothing
+				addlArgs := n.Policies
+				Expect(addlArgs).Should(HaveLen(1))
+			})
+		})
+
+		Context("via v2 api", func() {
+			var n NetConf
+			BeforeEach(func() {
+				n = NetConf{ApiVersion: 2}
+			})
+
+			It("append different IP", func() {
+				// mock different IP
+				n.ApplyDefaultPAPolicy("192.168.0.1")
+				n.ApplyDefaultPAPolicy("192.168.0.2")
+
+				// will be two policies
+				addlArgs := n.Policies
+				Expect(addlArgs).Should(HaveLen(2))
+
+				// normal type judgement
+				policy := addlArgs[1] // judge second item
+				Expect(policy.Name).Should(Equal("EndpointPolicy"))
+				value := make(map[string]interface{})
+				json.Unmarshal(policy.Value, &value)
+				Expect(value).Should(HaveKey("Type"))
+				Expect(value["Type"]).Should(Equal("ProviderAddress"))
+				Expect(value).Should(HaveKey("Settings"))
+
+				// compare with second item
+				settings := value["Settings"].(map[string]interface{})
+				paAddress := settings["ProviderAddress"].(string)
+				Expect(paAddress).Should(Equal("192.168.0.2"))
+			})
+
+			It("nothing to do if IP is blank", func() {
+				// mock different policy
+				n.Policies = []Policy{
+					{
+						Name:  "EndpointPolicy",
+						Value: bprintf(`{"Type": "OutBoundNAT", "Settings": {"Exceptions": ["192.168.0.0/16"]}}`),
+					},
+				}
+				n.ApplyDefaultPAPolicy("")
+
+				// nothing
+				addlArgs := n.Policies
+				Expect(addlArgs).Should(HaveLen(1))
 			})
 		})
 	})
 
 	Describe("ApplyPortMappingPolicy", func() {
-		Context("when portMappings not activated", func() {
+		Context("via v1 api", func() {
-			It("does nothing", func() {
+			var n NetConf
-				n := NetConf{}
+			BeforeEach(func() {
+				n = NetConf{}
+			})
+
+			It("nothing to do if input is empty", func() {
 				n.ApplyPortMappingPolicy(nil)
 				Expect(n.Policies).Should(BeNil())
 
 				n.ApplyPortMappingPolicy([]PortMapEntry{})
-				Expect(n.Policies).Should(HaveLen(0))
+				Expect(n.Policies).Should(BeNil())
-			})
 			})
 
-		Context("when portMappings is activated", func() {
+			It("create one NAT policy", func() {
-			It("creates NAT policies", func() {
+				// mock different IP
-				n := NetConf{}
 				n.ApplyPortMappingPolicy([]PortMapEntry{
 					{
 						ContainerPort: 80,
 						HostPort:      8080,
 						Protocol:      "TCP",
-						HostIP:        "ignored",
+						HostIP:        "192.168.1.2",
 					},
 				})
 
-				Expect(n.Policies).Should(HaveLen(1))
+				// only one item
+				addlArgs := n.Policies
+				Expect(addlArgs).Should(HaveLen(1))
 
-				policy := n.Policies[0]
+				// normal type judgement
+				policy := addlArgs[0]
 				Expect(policy.Name).Should(Equal("EndpointPolicy"))
-
 				value := make(map[string]interface{})
 				json.Unmarshal(policy.Value, &value)
-
 				Expect(value).Should(HaveKey("Type"))
 				Expect(value["Type"]).Should(Equal("NAT"))
 
+				// compare all values
 				Expect(value).Should(HaveKey("InternalPort"))
 				Expect(value["InternalPort"]).Should(Equal(float64(80)))
-
 				Expect(value).Should(HaveKey("ExternalPort"))
 				Expect(value["ExternalPort"]).Should(Equal(float64(8080)))
-
 				Expect(value).Should(HaveKey("Protocol"))
 				Expect(value["Protocol"]).Should(Equal("TCP"))
 			})
 		})
+
+		Context("via v2 api", func() {
+			var n NetConf
+			BeforeEach(func() {
+				n = NetConf{ApiVersion: 2}
 			})
 
-	Describe("MarshalPolicies", func() {
+			It("nothing to do if input is empty", func() {
-		Context("when not set by user", func() {
+				n.ApplyPortMappingPolicy(nil)
-			It("sets it by adding a policy", func() {
+				Expect(n.Policies).Should(BeNil())
 
-				n := NetConf{
+				n.ApplyPortMappingPolicy([]PortMapEntry{})
-					Policies: []policy{
+				Expect(n.Policies).Should(BeNil())
+			})
+
+			It("creates one NAT policy", func() {
+				// mock different IP
+				n.ApplyPortMappingPolicy([]PortMapEntry{
+					{
+						ContainerPort: 80,
+						HostPort:      8080,
+						Protocol:      "TCP",
+						HostIP:        "192.168.1.2",
+					},
+				})
+
+				// only one item
+				addlArgs := n.Policies
+				Expect(addlArgs).Should(HaveLen(1))
+
+				// normal type judgement
+				policy := addlArgs[0]
+				Expect(policy.Name).Should(Equal("EndpointPolicy"))
+				value := make(map[string]interface{})
+				json.Unmarshal(policy.Value, &value)
+				Expect(value).Should(HaveKey("Type"))
+				Expect(value["Type"]).Should(Equal("PortMapping"))
+				Expect(value).Should(HaveKey("Settings"))
+
+				// compare all values
+				settings := value["Settings"].(map[string]interface{})
+				Expect(settings).Should(HaveKey("InternalPort"))
+				Expect(settings["InternalPort"]).Should(Equal(float64(80)))
+				Expect(settings).Should(HaveKey("ExternalPort"))
+				Expect(settings["ExternalPort"]).Should(Equal(float64(8080)))
+				Expect(settings).Should(HaveKey("Protocol"))
+				Expect(settings["Protocol"]).Should(Equal(float64(6)))
+				Expect(settings).Should(HaveKey("VIP"))
+				Expect(settings["VIP"]).Should(Equal("192.168.1.2"))
+			})
+		})
+	})
+
+	Describe("GetXEndpointPolicies", func() {
+		Context("via v1 api", func() {
+			var n NetConf
+			BeforeEach(func() {
+				n = NetConf{}
+			})
+
+			It("GetHNSEndpointPolicies", func() {
+				// mock different policies
+				n.Policies = []Policy{
 					{
 						Name:  "EndpointPolicy",
-							Value: []byte(`{"someKey": "someValue"}`),
+						Value: []byte(`{"Type": "OutBoundNAT", "ExceptionList": [ "192.168.1.2" ]}`),
 					},
 					{
 						Name:  "someOtherType",
 						Value: []byte(`{"someOtherKey": "someOtherValue"}`),
 					},
-					},
 				}
 
-				result := n.MarshalPolicies()
+				// only one valid item
+				result := n.GetHNSEndpointPolicies()
 				Expect(len(result)).To(Equal(1))
 
+				// normal type judgement
 				policy := make(map[string]interface{})
 				err := json.Unmarshal(result[0], &policy)
 				Expect(err).ToNot(HaveOccurred())
-				Expect(policy).Should(HaveKey("someKey"))
+				Expect(policy).Should(HaveKey("Type"))
-				Expect(policy["someKey"]).To(Equal("someValue"))
+				Expect(policy["Type"]).To(Equal("OutBoundNAT"))
+				Expect(policy).Should(HaveKey("ExceptionList"))
+				Expect(policy["ExceptionList"]).To(ContainElement("192.168.1.2"))
 			})
 		})
 
-		Context("when set by user", func() {
+		Context("via v2 api", func() {
-			It("appends exceptions to the existing policy", func() {
+			var n NetConf
-				// first set it
+			BeforeEach(func() {
-				n := NetConf{}
+				n = NetConf{ApiVersion: 2}
-				n.ApplyOutboundNatPolicy("192.168.0.0/16")
+			})
 
-				// then attempt to update it
+			It("GetHostComputeEndpointPolicies", func() {
-				n.ApplyOutboundNatPolicy("10.244.0.0/16")
+				// mock different policies
+				n.Policies = []Policy{
+					{
+						Name:  "EndpointPolicy",
+						Value: []byte(`{"Type": "OutBoundNAT", "Settings": {"Exceptions": [ "192.168.1.2" ]}}`),
+					},
+					{
+						Name:  "someOtherType",
+						Value: []byte(`{"someOtherKey": "someOtherValue"}`),
+					},
+				}
 
-				// it should be unchanged!
+				// only one valid item
-				addlArgs := n.Policies
+				result := n.GetHostComputeEndpointPolicies()
-				Expect(addlArgs).Should(HaveLen(1))
+				Expect(len(result)).To(Equal(1))
 
-				policy := addlArgs[0]
+				// normal type judgement
-				Expect(policy.Name).Should(Equal("EndpointPolicy"))
+				policy := result[0]
-
+				Expect(policy.Type).Should(Equal(hcn.OutBoundNAT))
-				var value map[string]interface{}
+				settings := make(map[string]interface{})
-				json.Unmarshal(policy.Value, &value)
+				err := json.Unmarshal(policy.Settings, &settings)
-
+				Expect(err).ToNot(HaveOccurred())
-				Expect(value).Should(HaveKey("Type"))
+				Expect(settings["Exceptions"]).To(ContainElement("192.168.1.2"))
-				Expect(value).Should(HaveKey("ExceptionList"))
-				Expect(value["Type"]).Should(Equal("OutBoundNAT"))
-
-				exceptionList := value["ExceptionList"].([]interface{})
-				Expect(exceptionList).Should(HaveLen(2))
-				Expect(exceptionList[0].(string)).Should(Equal("192.168.0.0/16"))
-				Expect(exceptionList[1].(string)).Should(Equal("10.244.0.0/16"))
 			})
 		})
 	})

plugins/main/windows/win-bridge/sample-v2.conf

@@ -0,0 +1,52 @@
+{
+    "name":"cbr0",
+    "type":"flannel",
+    "delegate":{
+        "apiVersion":2,
+        "type":"win-bridge",
+        "dns":{
+            "nameservers":[
+                "11.0.0.10"
+            ],
+            "search":[
+                "svc.cluster.local"
+            ]
+        },
+        "policies":[
+            {
+                "name":"EndpointPolicy",
+                "value":{
+                    "Type":"OutBoundNAT",
+                    "Settings":{
+                        "Exceptions":[
+                            "192.168.0.0/16",
+                            "11.0.0.0/8",
+                            "10.137.196.0/23"
+                        ]
+                    }
+                }
+            },
+            {
+                "name":"EndpointPolicy",
+                "value":{
+                    "Type":"SDNRoute",
+                    "Settings":{
+                        "DestinationPrefix":"11.0.0.0/8",
+                        "NeedEncap":true
+                    }
+                }
+            },
+            {
+                "name":"EndpointPolicy",
+                "value":{
+                    "Type":"SDNRoute",
+                    "Settings":{
+                        "DestinationPrefix":"10.137.198.27/32",
+                        "NeedEncap":true
+                    }
+                }
+            }
+        ],
+        "loopbackDSR":true
+    }
+}

plugins/main/windows/win-bridge/win-bridge_windows.go

@@ -84,9 +84,7 @@ func processEndpointArgs(args *skel.CmdArgs, n *NetConf) (*hns.EndpointInfo, err
 	}
 
 	// configure sNAT exception
-	if len(n.IPMasqNetwork) != 0 {
 	n.ApplyOutboundNatPolicy(n.IPMasqNetwork)
-	}
 
 	// add port mapping if any present
 	n.ApplyPortMappingPolicy(n.RuntimeConfig.PortMaps)

plugins/main/windows/win-overlay/win-overlay_windows.go

@@ -119,7 +119,7 @@ func cmdAdd(args *skel.CmdArgs) error {
 
 		result.DNS = n.GetDNS()
 		if n.LoopbackDSR {
-			n.ApplyLoopbackDSR(&ipAddr)
+			n.ApplyLoopbackDSRPolicy(&ipAddr)
 		}
 		hnsEndpoint := &hcsshim.HNSEndpoint{
 			Name:           epName,
@@ -129,7 +129,7 @@ func cmdAdd(args *skel.CmdArgs) error {
 			GatewayAddress: gw,
 			IPAddress:      ipAddr,
 			MacAddress:     macAddr,
-			Policies:       n.MarshalPolicies(),
+			Policies:       n.GetHNSEndpointPolicies(),
 		}
 
 		return hnsEndpoint, nil