From 4b180a9d9c6416eaa2cf553a793b6b66d74d5769 Mon Sep 17 00:00:00 2001 From: thxcode Date: Mon, 19 Apr 2021 19:46:33 +0800 Subject: [PATCH] refactor(win-bridge): netconf - support v2 api - unify v1 and v2 api BREAKING CHANGE: - remove `HcnPolicyArgs` field - merge `HcnPolicyArgs` into `Policies` field Signed-off-by: thxcode --- pkg/hns/endpoint_windows.go | 17 +- pkg/hns/netconf_suite_windows_test.go | 4 +- pkg/hns/netconf_windows.go | 307 ++++++--- pkg/hns/netconf_windows_test.go | 598 ++++++++++++++---- .../{sample.conf => sample-v1.conf} | 0 .../main/windows/win-bridge/sample-v2.conf | 52 ++ .../windows/win-bridge/win-bridge_windows.go | 4 +- .../win-overlay/win-overlay_windows.go | 4 +- 8 files changed, 758 insertions(+), 228 deletions(-) rename plugins/main/windows/win-bridge/{sample.conf => sample-v1.conf} (100%) create mode 100644 plugins/main/windows/win-bridge/sample-v2.conf diff --git a/pkg/hns/endpoint_windows.go b/pkg/hns/endpoint_windows.go index 03aa0b77..681582c2 100644 --- a/pkg/hns/endpoint_windows.go +++ b/pkg/hns/endpoint_windows.go @@ -53,7 +53,7 @@ func GetSandboxContainerID(containerID string, netNs string) string { return containerID } -// GetIpString returns the given IP in string. +// GetIpString returns the given IP as a string. func GetIpString(ip *net.IP) string { if len(*ip) == 0 { return "" @@ -65,7 +65,7 @@ func GetIpString(ip *net.IP) string { // GetDefaultDestinationPrefix returns the default destination prefix according to the given IP type. func GetDefaultDestinationPrefix(ip *net.IP) string { destinationPrefix := "0.0.0.0/0" - if ipv6 := ip.To4(); ipv6 == nil { + if ip.To4() == nil { destinationPrefix = "::/0" } return destinationPrefix @@ -95,7 +95,7 @@ func GenerateHnsEndpoint(epInfo *EndpointInfo, n *NetConf) (*hcsshim.HNSEndpoint } if n.LoopbackDSR { - n.ApplyLoopbackDSR(&epInfo.IpAddress) + n.ApplyLoopbackDSRPolicy(&epInfo.IpAddress) } hnsEndpoint = &hcsshim.HNSEndpoint{ Name: epInfo.EndpointName, @@ -104,7 +104,7 @@ func GenerateHnsEndpoint(epInfo *EndpointInfo, n *NetConf) (*hcsshim.HNSEndpoint DNSSuffix: strings.Join(epInfo.DNS.Search, ","), GatewayAddress: GetIpString(&epInfo.Gateway), IPAddress: epInfo.IpAddress, - Policies: n.MarshalPolicies(), + Policies: n.GetHNSEndpointPolicies(), } return hnsEndpoint, nil } @@ -240,7 +240,7 @@ func GenerateHcnEndpoint(epInfo *EndpointInfo, n *NetConf) (*hcn.HostComputeEndp } if n.LoopbackDSR { - n.ApplyLoopbackDSR(&epInfo.IpAddress) + n.ApplyLoopbackDSRPolicy(&epInfo.IpAddress) } hcnEndpoint = &hcn.HostComputeEndpoint{ SchemaVersion: hcn.SchemaVersion{ @@ -266,12 +266,7 @@ func GenerateHcnEndpoint(epInfo *EndpointInfo, n *NetConf) (*hcn.HostComputeEndp IpAddress: GetIpString(&epInfo.IpAddress), }, }, - Policies: func() []hcn.EndpointPolicy { - if n.HcnPolicyArgs == nil { - n.HcnPolicyArgs = []hcn.EndpointPolicy{} - } - return n.HcnPolicyArgs - }(), + Policies: n.GetHostComputeEndpointPolicies(), } return hcnEndpoint, nil } diff --git a/pkg/hns/netconf_suite_windows_test.go b/pkg/hns/netconf_suite_windows_test.go index cc69e6fd..90675961 100644 --- a/pkg/hns/netconf_suite_windows_test.go +++ b/pkg/hns/netconf_suite_windows_test.go @@ -20,7 +20,7 @@ import ( "testing" ) -func TestHns(t *testing.T) { +func TestNetConf(t *testing.T) { RegisterFailHandler(Fail) - RunSpecs(t, "HNS NetConf Suite") + RunSpecs(t, "NetConf Suite") } diff --git a/pkg/hns/netconf_windows.go b/pkg/hns/netconf_windows.go index 897a1528..cae0589f 100644 --- a/pkg/hns/netconf_windows.go +++ b/pkg/hns/netconf_windows.go @@ -17,8 +17,10 @@ package hns import ( "bytes" "encoding/json" + "errors" "fmt" "net" + "strconv" "strings" "github.com/Microsoft/hcsshim/hcn" @@ -29,16 +31,16 @@ import ( // NetConf is the CNI spec type NetConf struct { types.NetConf - // ApiVersion is either 1 or 2, which specifies which hns APIs to call - ApiVersion int `json:"ApiVersion"` - // V2 Api Policies - HcnPolicyArgs []hcn.EndpointPolicy `json:"HcnPolicyArgs,omitempty"` - // V1 Api Policies - Policies []policy `json:"policies,omitempty"` - // Options to be passed in by the runtime + // ApiVersion specifies the policies type of HNS or HCN, select one of [1, 2]. + // HNS is the v1 API, which is the default version and applies to dockershim. + // HCN is the v2 API, which can leverage HostComputeNamespace and use in containerd. + ApiVersion int `json:"apiVersion,omitempty"` + // Policies specifies the policy list for HNSEndpoint or HostComputeEndpoint. + Policies []Policy `json:"policies,omitempty"` + // RuntimeConfig represents the options to be passed in by the runtime. RuntimeConfig RuntimeConfig `json:"runtimeConfig"` - // If true, adds a policy to endpoints to support loopback direct server return - LoopbackDSR bool `json:"loopbackDSR"` + // LoopbackDSR specifies whether to support loopback direct server return. + LoopbackDSR bool `json:"loopbackDSR,omitempty"` } type RuntimeDNS struct { @@ -53,32 +55,63 @@ type PortMapEntry struct { HostIP string `json:"hostIP,omitempty"` } +// constants of the supported Windows Socket protocol, +// ref to https://docs.microsoft.com/en-us/dotnet/api/system.net.sockets.protocoltype. +var protocolEnums = map[string]uint32{ + "icmpv4": 1, + "igmp": 2, + "tcp": 6, + "udp": 17, + "icmpv6": 58, +} + +func (p *PortMapEntry) GetProtocolEnum() (uint32, error) { + var u, err = strconv.ParseUint(p.Protocol, 0, 10) + if err != nil { + var pe, exist = protocolEnums[strings.ToLower(p.Protocol)] + if !exist { + return 0, errors.New("invalid protocol supplied to port mapping policy") + } + return pe, nil + } + return uint32(u), nil +} + type RuntimeConfig struct { DNS RuntimeDNS `json:"dns"` PortMaps []PortMapEntry `json:"portMappings,omitempty"` } -type policy struct { +type Policy struct { Name string `json:"name"` Value json.RawMessage `json:"value"` } -// MarshalPolicies converts the HNSEndpoint policies in Policies -// to HNS specific policies as Json raw bytes. -func (n *NetConf) MarshalPolicies() []json.RawMessage { - if n.Policies == nil { - n.Policies = make([]policy, 0) - } - +// GetHNSEndpointPolicies converts the configuration policies to HNSEndpoint policies. +func (n *NetConf) GetHNSEndpointPolicies() []json.RawMessage { result := make([]json.RawMessage, 0, len(n.Policies)) for _, p := range n.Policies { if !strings.EqualFold(p.Name, "EndpointPolicy") { continue } - result = append(result, p.Value) } + return result +} +// GetHostComputeEndpointPolicies converts the configuration policies to HostComputeEndpoint policies. +func (n *NetConf) GetHostComputeEndpointPolicies() []hcn.EndpointPolicy { + result := make([]hcn.EndpointPolicy, 0, len(n.Policies)) + for _, p := range n.Policies { + if !strings.EqualFold(p.Name, "EndpointPolicy") { + continue + } + var policy hcn.EndpointPolicy + if err := json.Unmarshal(p.Value, &policy); err != nil { + continue + } + result = append(result, policy) + } return result } @@ -94,134 +127,222 @@ func (n *NetConf) GetDNS() types.DNS { return dnsResult } -// ApplyLoopbackDSR configures the given IP to support loopback DSR. -func (n *NetConf) ApplyLoopbackDSR(ip *net.IP) { - value := fmt.Sprintf(`"Destinations" : ["%s"]`, ip.String()) - if n.ApiVersion == 2 { - hcnLoopbackRoute := hcn.EndpointPolicy{ - Type: "OutBoundNAT", - Settings: []byte(fmt.Sprintf("{%s}", value)), - } - n.HcnPolicyArgs = append(n.HcnPolicyArgs, hcnLoopbackRoute) - } else { - hnsLoopbackRoute := policy{ - Name: "EndpointPolicy", - Value: []byte(fmt.Sprintf(`{"Type": "OutBoundNAT", %s}`, value)), - } - n.Policies = append(n.Policies, hnsLoopbackRoute) - } -} - -// ApplyOutboundNatPolicy applies the sNAT policy in HNS/HCN and configures the given CIDR as an exception. -func (n *NetConf) ApplyOutboundNatPolicy(nwToNat string) { - if n.Policies == nil { - n.Policies = make([]policy, 0) +// ApplyLoopbackDSRPolicy configures the given IP to support loopback DSR. +func (n *NetConf) ApplyLoopbackDSRPolicy(ip *net.IP) { + if err := hcn.DSRSupported(); err != nil || ip == nil { + return } - nwToNatBytes := []byte(nwToNat) + toPolicyValue := func(addr string) json.RawMessage { + if n.ApiVersion == 2 { + return bprintf(`{"Type": "OutBoundNAT", "Settings": {"Destinations": ["%s"]}}`, addr) + } + return bprintf(`{"Type": "OutBoundNAT", "Destinations": ["%s"]}`, addr) + } + ipBytes := []byte(ip.String()) - for i, p := range n.Policies { + // find OutBoundNAT policy + for i := range n.Policies { + p := &n.Policies[i] if !strings.EqualFold(p.Name, "EndpointPolicy") { continue } - typeValue, err := jsonparser.GetUnsafeString(p.Value, "Type") - if err != nil || len(typeValue) == 0 { + // filter OutBoundNAT policy + typeValue, _ := jsonparser.GetUnsafeString(p.Value, "Type") + if typeValue != "OutBoundNAT" { continue } - if !strings.EqualFold(typeValue, "OutBoundNAT") { + // parse destination address list + var ( + destinationsValue []byte + dt jsonparser.ValueType + ) + if n.ApiVersion == 2 { + destinationsValue, dt, _, _ = jsonparser.Get(p.Value, "Settings", "Destinations") + } else { + destinationsValue, dt, _, _ = jsonparser.Get(p.Value, "Destinations") + } + + // skip if Destinations/DestinationList field is not found + if dt == jsonparser.NotExist { continue } - exceptionListValue, dt, _, _ := jsonparser.Get(p.Value, "ExceptionList") - // OutBoundNAT must with ExceptionList, so don't need to judge jsonparser.NotExist + // return if found the given address if dt == jsonparser.Array { - buf := bytes.Buffer{} - buf.WriteString(`{"Type": "OutBoundNAT", "ExceptionList": [`) - - jsonparser.ArrayEach(exceptionListValue, func(value []byte, dataType jsonparser.ValueType, offset int, err error) { + var found bool + _, _ = jsonparser.ArrayEach(destinationsValue, func(value []byte, dataType jsonparser.ValueType, offset int, err error) { if dataType == jsonparser.String && len(value) != 0 { - if bytes.Compare(value, nwToNatBytes) != 0 { - buf.WriteByte('"') - buf.Write(value) - buf.WriteByte('"') - buf.WriteByte(',') + if bytes.Compare(value, ipBytes) == 0 { + found = true } } }) - - buf.WriteString(`"` + nwToNat + `"]}`) - - n.Policies[i] = policy{ - Name: "EndpointPolicy", - Value: buf.Bytes(), - } - } else { - n.Policies[i] = policy{ - Name: "EndpointPolicy", - Value: []byte(`{"Type": "OutBoundNAT", "ExceptionList": ["` + nwToNat + `"]}`), + if found { + return } } + } + // or add a new OutBoundNAT if not found + n.Policies = append(n.Policies, Policy{ + Name: "EndpointPolicy", + Value: toPolicyValue(ip.String()), + }) +} + +// ApplyOutboundNatPolicy applies the sNAT policy in HNS/HCN and configures the given CIDR as an exception. +func (n *NetConf) ApplyOutboundNatPolicy(exceptionCIDR string) { + if exceptionCIDR == "" { return } - // didn't find the policyArg, add it - n.Policies = append(n.Policies, policy{ + toPolicyValue := func(cidr ...string) json.RawMessage { + if n.ApiVersion == 2 { + return bprintf(`{"Type": "OutBoundNAT", "Settings": {"Exceptions": ["%s"]}}`, strings.Join(cidr, `","`)) + } + return bprintf(`{"Type": "OutBoundNAT", "ExceptionList": ["%s"]}`, strings.Join(cidr, `","`)) + } + exceptionCIDRBytes := []byte(exceptionCIDR) + + // find OutBoundNAT policy + for i := range n.Policies { + p := &n.Policies[i] + if !strings.EqualFold(p.Name, "EndpointPolicy") { + continue + } + + // filter OutBoundNAT policy + typeValue, _ := jsonparser.GetUnsafeString(p.Value, "Type") + if typeValue != "OutBoundNAT" { + continue + } + + // parse exception CIDR list + var ( + exceptionsValue []byte + dt jsonparser.ValueType + ) + if n.ApiVersion == 2 { + exceptionsValue, dt, _, _ = jsonparser.Get(p.Value, "Settings", "Exceptions") + } else { + exceptionsValue, dt, _, _ = jsonparser.Get(p.Value, "ExceptionList") + } + + // skip if Exceptions/ExceptionList field is not found + if dt == jsonparser.NotExist { + continue + } + + // return if found the given CIDR + if dt == jsonparser.Array { + var found bool + _, _ = jsonparser.ArrayEach(exceptionsValue, func(value []byte, dataType jsonparser.ValueType, offset int, err error) { + if dataType == jsonparser.String && len(value) != 0 { + if bytes.Compare(value, exceptionCIDRBytes) == 0 { + found = true + } + } + }) + if found { + return + } + } + } + + // or add a new OutBoundNAT if not found + n.Policies = append(n.Policies, Policy{ Name: "EndpointPolicy", - Value: []byte(`{"Type": "OutBoundNAT", "ExceptionList": ["` + nwToNat + `"]}`), + Value: toPolicyValue(exceptionCIDR), }) } // ApplyDefaultPAPolicy applies an endpoint PA policy in HNS/HCN. -func (n *NetConf) ApplyDefaultPAPolicy(paAddress string) { - if n.Policies == nil { - n.Policies = make([]policy, 0) +func (n *NetConf) ApplyDefaultPAPolicy(address string) { + if address == "" { + return } - // if its already present, leave untouched - for i, p := range n.Policies { + toPolicyValue := func(addr string) json.RawMessage { + if n.ApiVersion == 2 { + return bprintf(`{"Type": "ProviderAddress", "Settings": {"ProviderAddress": "%s"}}`, addr) + } + return bprintf(`{"Type": "PA", "PA": "%s"}`, addr) + } + addressBytes := []byte(address) + + // find ProviderAddress policy + for i := range n.Policies { + p := &n.Policies[i] if !strings.EqualFold(p.Name, "EndpointPolicy") { continue } - paValue, dt, _, _ := jsonparser.Get(p.Value, "PA") + // filter ProviderAddress policy + typeValue, _ := jsonparser.GetUnsafeString(p.Value, "Type") + if typeValue != "PA" && typeValue != "ProviderAddress" { + continue + } + + // parse provider address + var ( + paValue []byte + dt jsonparser.ValueType + ) + if n.ApiVersion == 2 { + paValue, dt, _, _ = jsonparser.Get(p.Value, "Settings", "ProviderAddress") + } else { + paValue, dt, _, _ = jsonparser.Get(p.Value, "PA") + } + + // skip if ProviderAddress/PA field is not found if dt == jsonparser.NotExist { continue - } else if dt == jsonparser.String && len(paValue) != 0 { - // found it, don't override + } + + // return if found the given address + if dt == jsonparser.String && bytes.Compare(paValue, addressBytes) == 0 { return } - - n.Policies[i] = policy{ - Name: "EndpointPolicy", - Value: []byte(`{"Type": "PA", "PA": "` + paAddress + `"}`), - } - return } - // didn't find the policyArg, add it - n.Policies = append(n.Policies, policy{ + // or add a new ProviderAddress if not found + n.Policies = append(n.Policies, Policy{ Name: "EndpointPolicy", - Value: []byte(`{"Type": "PA", "PA": "` + paAddress + `"}`), + Value: toPolicyValue(address), }) } // ApplyPortMappingPolicy applies the host/container port mapping policies in HNS/HCN. func (n *NetConf) ApplyPortMappingPolicy(portMappings []PortMapEntry) { - if portMappings == nil { + if len(portMappings) == 0 { return } - if n.Policies == nil { - n.Policies = make([]policy, 0) + toPolicyValue := func(p *PortMapEntry) json.RawMessage { + if n.ApiVersion == 2 { + var protocolEnum, _ = p.GetProtocolEnum() + return bprintf(`{"Type": "PortMapping", "Settings": {"InternalPort": %d, "ExternalPort": %d, "Protocol": %d, "VIP": "%s"}}`, p.ContainerPort, p.HostPort, protocolEnum, p.HostIP) + } + return bprintf(`{"Type": "NAT", "InternalPort": %d, "ExternalPort": %d, "Protocol": "%s"}`, p.ContainerPort, p.HostPort, p.Protocol) } - for _, portMapping := range portMappings { - n.Policies = append(n.Policies, policy{ + for i := range portMappings { + p := &portMappings[i] + // skip the invalid protocol mapping + if _, err := p.GetProtocolEnum(); err != nil { + continue + } + n.Policies = append(n.Policies, Policy{ Name: "EndpointPolicy", - Value: []byte(fmt.Sprintf(`{"Type": "NAT", "InternalPort": %d, "ExternalPort": %d, "Protocol": "%s"}`, portMapping.ContainerPort, portMapping.HostPort, portMapping.Protocol)), + Value: toPolicyValue(p), }) } } + +// bprintf is similar to fmt.Sprintf and returns a byte array as result. +func bprintf(format string, a ...interface{}) []byte { + return []byte(fmt.Sprintf(format, a...)) +} diff --git a/pkg/hns/netconf_windows_test.go b/pkg/hns/netconf_windows_test.go index 4bfa18af..3d2d9a66 100644 --- a/pkg/hns/netconf_windows_test.go +++ b/pkg/hns/netconf_windows_test.go @@ -15,221 +15,585 @@ package hns import ( "encoding/json" + "net" + "github.com/Microsoft/hcsshim/hcn" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" ) -var _ = Describe("HNS NetConf", func() { - Describe("ApplyOutBoundNATPolicy", func() { - Context("when not set by user", func() { - It("sets it by adding a policy", func() { +var _ = Describe("NetConf", func() { + Describe("ApplyLoopbackDSRPolicy", func() { + Context("via v1 api", func() { + var n NetConf + BeforeEach(func() { + n = NetConf{} + }) - // apply it - n := NetConf{} - n.ApplyOutboundNatPolicy("192.168.0.0/16") + It("filter out duplicated IP", func() { + // mock duplicated IP + ip := net.ParseIP("172.16.0.12") + n.ApplyLoopbackDSRPolicy(&ip) + n.ApplyLoopbackDSRPolicy(&ip) + // only one policy addlArgs := n.Policies Expect(addlArgs).Should(HaveLen(1)) + // normal type judgement policy := addlArgs[0] Expect(policy.Name).Should(Equal("EndpointPolicy")) - value := make(map[string]interface{}) json.Unmarshal(policy.Value, &value) - Expect(value).Should(HaveKey("Type")) - Expect(value).Should(HaveKey("ExceptionList")) Expect(value["Type"]).Should(Equal("OutBoundNAT")) + Expect(value).Should(HaveKey("Destinations")) - exceptionList := value["ExceptionList"].([]interface{}) - Expect(exceptionList).Should(HaveLen(1)) - Expect(exceptionList[0].(string)).Should(Equal("192.168.0.0/16")) + // and only one item + destinationList := value["Destinations"].([]interface{}) + Expect(destinationList).Should(HaveLen(1)) + Expect(destinationList[0].(string)).Should(Equal("172.16.0.12")) + }) + + It("append different IP", func() { + // mock different IP + ip1 := net.ParseIP("172.16.0.12") + n.ApplyLoopbackDSRPolicy(&ip1) + ip2 := net.ParseIP("172.16.0.13") + n.ApplyLoopbackDSRPolicy(&ip2) + + // will be two policies + addlArgs := n.Policies + Expect(addlArgs).Should(HaveLen(2)) + + // normal type judgement + policy := addlArgs[1] // pick second item + Expect(policy.Name).Should(Equal("EndpointPolicy")) + value := make(map[string]interface{}) + json.Unmarshal(policy.Value, &value) + Expect(value).Should(HaveKey("Type")) + Expect(value["Type"]).Should(Equal("OutBoundNAT")) + Expect(value).Should(HaveKey("Destinations")) + + // only one item + destinationList := value["Destinations"].([]interface{}) + Expect(destinationList).Should(HaveLen(1)) + Expect(destinationList[0].(string)).Should(Equal("172.16.0.13")) }) }) - Context("when set by user", func() { - It("appends exceptions to the existing policy", func() { - // first set it - n := NetConf{} - n.ApplyOutboundNatPolicy("192.168.0.0/16") + Context("via v2 api", func() { + var n NetConf + BeforeEach(func() { + n = NetConf{ApiVersion: 2} + }) - // then attempt to update it - n.ApplyOutboundNatPolicy("10.244.0.0/16") + It("filter out duplicated IP", func() { + // mock duplicated IP + ip := net.ParseIP("172.16.0.12") + n.ApplyLoopbackDSRPolicy(&ip) + n.ApplyLoopbackDSRPolicy(&ip) - // it should be unchanged! + // only one policy addlArgs := n.Policies Expect(addlArgs).Should(HaveLen(1)) + // normal type judgement policy := addlArgs[0] Expect(policy.Name).Should(Equal("EndpointPolicy")) - - var value map[string]interface{} + value := make(map[string]interface{}) json.Unmarshal(policy.Value, &value) - Expect(value).Should(HaveKey("Type")) - Expect(value).Should(HaveKey("ExceptionList")) Expect(value["Type"]).Should(Equal("OutBoundNAT")) + Expect(value).Should(HaveKey("Settings")) + // and only one item + settings := value["Settings"].(map[string]interface{}) + destinationList := settings["Destinations"].([]interface{}) + Expect(destinationList).Should(HaveLen(1)) + Expect(destinationList[0].(string)).Should(Equal("172.16.0.12")) + }) + + It("append different IP", func() { + // mock different IP + ip1 := net.ParseIP("172.16.0.12") + n.ApplyLoopbackDSRPolicy(&ip1) + ip2 := net.ParseIP("172.16.0.13") + n.ApplyLoopbackDSRPolicy(&ip2) + + // will be two policies + addlArgs := n.Policies + Expect(addlArgs).Should(HaveLen(2)) + + // normal type judgement + policy := addlArgs[1] // pick second item + Expect(policy.Name).Should(Equal("EndpointPolicy")) + value := make(map[string]interface{}) + json.Unmarshal(policy.Value, &value) + Expect(value).Should(HaveKey("Type")) + Expect(value["Type"]).Should(Equal("OutBoundNAT")) + Expect(value).Should(HaveKey("Settings")) + + // only one item + settings := value["Settings"].(map[string]interface{}) + destinationList := settings["Destinations"].([]interface{}) + Expect(destinationList).Should(HaveLen(1)) + Expect(destinationList[0].(string)).Should(Equal("172.16.0.13")) + }) + }) + }) + + Describe("ApplyOutBoundNATPolicy", func() { + Context("via v1 api", func() { + var n NetConf + BeforeEach(func() { + n = NetConf{} + }) + + It("append different IP", func() { + // mock different IP + n.ApplyOutboundNatPolicy("192.168.0.0/16") + n.ApplyOutboundNatPolicy("10.244.0.0/16") + + // will be two policies + addlArgs := n.Policies + Expect(addlArgs).Should(HaveLen(2)) + + // normal type judgement + policy := addlArgs[1] // pick second item + Expect(policy.Name).Should(Equal("EndpointPolicy")) + value := make(map[string]interface{}) + json.Unmarshal(policy.Value, &value) + Expect(value).Should(HaveKey("Type")) + Expect(value["Type"]).Should(Equal("OutBoundNAT")) + Expect(value).Should(HaveKey("ExceptionList")) + + // but get two items exceptionList := value["ExceptionList"].([]interface{}) - Expect(exceptionList).Should(HaveLen(2)) - Expect(exceptionList[0].(string)).Should(Equal("192.168.0.0/16")) - Expect(exceptionList[1].(string)).Should(Equal("10.244.0.0/16")) + Expect(exceptionList).Should(HaveLen(1)) + Expect(exceptionList[0].(string)).Should(Equal("10.244.0.0/16")) + }) + + It("append a new one if there is not an exception OutBoundNAT policy", func() { + // mock different OutBoundNAT routes + n.Policies = []Policy{ + { + Name: "EndpointPolicy", + Value: bprintf(`{"Type": "OutBoundNAT", "OtherList": []}`), + }, + } + n.ApplyOutboundNatPolicy("10.244.0.0/16") + + // has two policies + addlArgs := n.Policies + Expect(addlArgs).Should(HaveLen(2)) + + // normal type judgement + policy := addlArgs[0] + Expect(policy.Name).Should(Equal("EndpointPolicy")) + value := make(map[string]interface{}) + json.Unmarshal(policy.Value, &value) + Expect(value).Should(HaveKey("Type")) + Expect(value["Type"]).Should(Equal("OutBoundNAT")) + Expect(value).Should(HaveKey("OtherList")) + policy = addlArgs[1] + value = make(map[string]interface{}) + json.Unmarshal(policy.Value, &value) + Expect(value).Should(HaveKey("Type")) + Expect(value["Type"]).Should(Equal("OutBoundNAT")) + Expect(value).Should(HaveKey("ExceptionList")) + + // only get one item + exceptionList := value["ExceptionList"].([]interface{}) + Expect(exceptionList).Should(HaveLen(1)) + Expect(exceptionList[0].(string)).Should(Equal("10.244.0.0/16")) + }) + + It("nothing to do if CIDR is blank", func() { + // mock different OutBoundNAT routes + n.Policies = []Policy{ + { + Name: "EndpointPolicy", + Value: bprintf(`{"Type": "OutBoundNAT", "ExceptionList": []}`), + }, + } + n.ApplyOutboundNatPolicy("") + + // only one policy + addlArgs := n.Policies + Expect(addlArgs).Should(HaveLen(1)) + + // normal type judgement + policy := addlArgs[0] + Expect(policy.Name).Should(Equal("EndpointPolicy")) + value := make(map[string]interface{}) + json.Unmarshal(policy.Value, &value) + Expect(value).Should(HaveKey("Type")) + Expect(value["Type"]).Should(Equal("OutBoundNAT")) + Expect(value).Should(HaveKey("ExceptionList")) + + // empty list + Expect(value["ExceptionList"]).ShouldNot(BeNil()) + Expect(value["ExceptionList"]).Should(HaveLen(0)) + }) + }) + + Context("via v2 api", func() { + var n NetConf + BeforeEach(func() { + n = NetConf{ApiVersion: 2} + }) + + It("append different IP", func() { + // mock different IP + n.ApplyOutboundNatPolicy("192.168.0.0/16") + n.ApplyOutboundNatPolicy("10.244.0.0/16") + + // will be two policies + addlArgs := n.Policies + Expect(addlArgs).Should(HaveLen(2)) + + // normal type judgement + policy := addlArgs[1] // pick second item + Expect(policy.Name).Should(Equal("EndpointPolicy")) + value := make(map[string]interface{}) + json.Unmarshal(policy.Value, &value) + Expect(value).Should(HaveKey("Type")) + Expect(value["Type"]).Should(Equal("OutBoundNAT")) + Expect(value).Should(HaveKey("Settings")) + + // but get two items + settings := value["Settings"].(map[string]interface{}) + exceptionList := settings["Exceptions"].([]interface{}) + Expect(exceptionList).Should(HaveLen(1)) + Expect(exceptionList[0].(string)).Should(Equal("10.244.0.0/16")) + }) + + It("append a new one if there is not an exception OutBoundNAT policy", func() { + // mock different OutBoundNAT routes + n.Policies = []Policy{ + { + Name: "EndpointPolicy", + Value: bprintf(`{"Type": "OutBoundNAT", "Settings": {"Others": []}}`), + }, + } + n.ApplyOutboundNatPolicy("10.244.0.0/16") + + // has two policies + addlArgs := n.Policies + Expect(addlArgs).Should(HaveLen(2)) + + // normal type judgement + policy := addlArgs[0] + Expect(policy.Name).Should(Equal("EndpointPolicy")) + value := make(map[string]interface{}) + json.Unmarshal(policy.Value, &value) + Expect(value).Should(HaveKey("Type")) + Expect(value["Type"]).Should(Equal("OutBoundNAT")) + Expect(value).Should(HaveKey("Settings")) + Expect(value["Settings"]).Should(HaveKey("Others")) + policy = addlArgs[1] + value = make(map[string]interface{}) + json.Unmarshal(policy.Value, &value) + Expect(value).Should(HaveKey("Type")) + Expect(value["Type"]).Should(Equal("OutBoundNAT")) + Expect(value).Should(HaveKey("Settings")) + + // only get one item + settings := value["Settings"].(map[string]interface{}) + exceptionList := settings["Exceptions"].([]interface{}) + Expect(exceptionList).Should(HaveLen(1)) + Expect(exceptionList[0].(string)).Should(Equal("10.244.0.0/16")) + }) + + It("nothing to do if CIDR is blank", func() { + // mock different OutBoundNAT routes + n.Policies = []Policy{ + { + Name: "EndpointPolicy", + Value: bprintf(`{"Type": "OutBoundNAT", "Settings": {"Exceptions": []}}`), + }, + } + n.ApplyOutboundNatPolicy("") + + // only one policy + addlArgs := n.Policies + Expect(addlArgs).Should(HaveLen(1)) + + // normal type judgement + policy := addlArgs[0] + Expect(policy.Name).Should(Equal("EndpointPolicy")) + value := make(map[string]interface{}) + json.Unmarshal(policy.Value, &value) + Expect(value).Should(HaveKey("Type")) + Expect(value["Type"]).Should(Equal("OutBoundNAT")) + Expect(value).Should(HaveKey("Settings")) + + // empty list + settings := value["Settings"].(map[string]interface{}) + Expect(settings["Exceptions"]).ShouldNot(BeNil()) + Expect(settings["Exceptions"]).Should(HaveLen(0)) }) }) }) Describe("ApplyDefaultPAPolicy", func() { - Context("when not set by user", func() { - It("sets it by adding a policy", func() { - - n := NetConf{} - n.ApplyDefaultPAPolicy("192.168.0.1") - - addlArgs := n.Policies - Expect(addlArgs).Should(HaveLen(1)) - - policy := addlArgs[0] - Expect(policy.Name).Should(Equal("EndpointPolicy")) - - value := make(map[string]interface{}) - json.Unmarshal(policy.Value, &value) - - Expect(value).Should(HaveKey("Type")) - Expect(value["Type"]).Should(Equal("PA")) - - paAddress := value["PA"].(string) - Expect(paAddress).Should(Equal("192.168.0.1")) + Context("via v1 api", func() { + var n NetConf + BeforeEach(func() { + n = NetConf{} }) - }) - Context("when set by user", func() { - It("does not override", func() { - n := NetConf{} + It("append different IP", func() { + // mock different IP n.ApplyDefaultPAPolicy("192.168.0.1") n.ApplyDefaultPAPolicy("192.168.0.2") + // will be two policies addlArgs := n.Policies - Expect(addlArgs).Should(HaveLen(1)) + Expect(addlArgs).Should(HaveLen(2)) - policy := addlArgs[0] + // normal type judgement + policy := addlArgs[1] // judge second item Expect(policy.Name).Should(Equal("EndpointPolicy")) - value := make(map[string]interface{}) json.Unmarshal(policy.Value, &value) - Expect(value).Should(HaveKey("Type")) Expect(value["Type"]).Should(Equal("PA")) + // compare with second item paAddress := value["PA"].(string) - Expect(paAddress).Should(Equal("192.168.0.1")) - Expect(paAddress).ShouldNot(Equal("192.168.0.2")) + Expect(paAddress).Should(Equal("192.168.0.2")) + }) + + It("nothing to do if IP is blank", func() { + // mock different policy + n.Policies = []Policy{ + { + Name: "EndpointPolicy", + Value: bprintf(`{"Type": "OutBoundNAT", "Exceptions": ["192.168.0.0/16"]}`), + }, + } + n.ApplyDefaultPAPolicy("") + + // nothing + addlArgs := n.Policies + Expect(addlArgs).Should(HaveLen(1)) + }) + }) + + Context("via v2 api", func() { + var n NetConf + BeforeEach(func() { + n = NetConf{ApiVersion: 2} + }) + + It("append different IP", func() { + // mock different IP + n.ApplyDefaultPAPolicy("192.168.0.1") + n.ApplyDefaultPAPolicy("192.168.0.2") + + // will be two policies + addlArgs := n.Policies + Expect(addlArgs).Should(HaveLen(2)) + + // normal type judgement + policy := addlArgs[1] // judge second item + Expect(policy.Name).Should(Equal("EndpointPolicy")) + value := make(map[string]interface{}) + json.Unmarshal(policy.Value, &value) + Expect(value).Should(HaveKey("Type")) + Expect(value["Type"]).Should(Equal("ProviderAddress")) + Expect(value).Should(HaveKey("Settings")) + + // compare with second item + settings := value["Settings"].(map[string]interface{}) + paAddress := settings["ProviderAddress"].(string) + Expect(paAddress).Should(Equal("192.168.0.2")) + }) + + It("nothing to do if IP is blank", func() { + // mock different policy + n.Policies = []Policy{ + { + Name: "EndpointPolicy", + Value: bprintf(`{"Type": "OutBoundNAT", "Settings": {"Exceptions": ["192.168.0.0/16"]}}`), + }, + } + n.ApplyDefaultPAPolicy("") + + // nothing + addlArgs := n.Policies + Expect(addlArgs).Should(HaveLen(1)) }) }) }) Describe("ApplyPortMappingPolicy", func() { - Context("when portMappings not activated", func() { - It("does nothing", func() { - n := NetConf{} + Context("via v1 api", func() { + var n NetConf + BeforeEach(func() { + n = NetConf{} + }) + + It("nothing to do if input is empty", func() { n.ApplyPortMappingPolicy(nil) Expect(n.Policies).Should(BeNil()) n.ApplyPortMappingPolicy([]PortMapEntry{}) - Expect(n.Policies).Should(HaveLen(0)) + Expect(n.Policies).Should(BeNil()) }) - }) - Context("when portMappings is activated", func() { - It("creates NAT policies", func() { - n := NetConf{} + It("create one NAT policy", func() { + // mock different IP n.ApplyPortMappingPolicy([]PortMapEntry{ { ContainerPort: 80, HostPort: 8080, Protocol: "TCP", - HostIP: "ignored", + HostIP: "192.168.1.2", }, }) - Expect(n.Policies).Should(HaveLen(1)) + // only one item + addlArgs := n.Policies + Expect(addlArgs).Should(HaveLen(1)) - policy := n.Policies[0] + // normal type judgement + policy := addlArgs[0] Expect(policy.Name).Should(Equal("EndpointPolicy")) - value := make(map[string]interface{}) json.Unmarshal(policy.Value, &value) - Expect(value).Should(HaveKey("Type")) Expect(value["Type"]).Should(Equal("NAT")) + // compare all values Expect(value).Should(HaveKey("InternalPort")) Expect(value["InternalPort"]).Should(Equal(float64(80))) - Expect(value).Should(HaveKey("ExternalPort")) Expect(value["ExternalPort"]).Should(Equal(float64(8080))) - Expect(value).Should(HaveKey("Protocol")) Expect(value["Protocol"]).Should(Equal("TCP")) }) }) - }) - Describe("MarshalPolicies", func() { - Context("when not set by user", func() { - It("sets it by adding a policy", func() { - - n := NetConf{ - Policies: []policy{ - { - Name: "EndpointPolicy", - Value: []byte(`{"someKey": "someValue"}`), - }, - { - Name: "someOtherType", - Value: []byte(`{"someOtherKey": "someOtherValue"}`), - }, - }, - } - - result := n.MarshalPolicies() - Expect(len(result)).To(Equal(1)) - - policy := make(map[string]interface{}) - err := json.Unmarshal(result[0], &policy) - Expect(err).ToNot(HaveOccurred()) - Expect(policy).Should(HaveKey("someKey")) - Expect(policy["someKey"]).To(Equal("someValue")) + Context("via v2 api", func() { + var n NetConf + BeforeEach(func() { + n = NetConf{ApiVersion: 2} }) - }) - Context("when set by user", func() { - It("appends exceptions to the existing policy", func() { - // first set it - n := NetConf{} - n.ApplyOutboundNatPolicy("192.168.0.0/16") + It("nothing to do if input is empty", func() { + n.ApplyPortMappingPolicy(nil) + Expect(n.Policies).Should(BeNil()) - // then attempt to update it - n.ApplyOutboundNatPolicy("10.244.0.0/16") + n.ApplyPortMappingPolicy([]PortMapEntry{}) + Expect(n.Policies).Should(BeNil()) + }) - // it should be unchanged! + It("creates one NAT policy", func() { + // mock different IP + n.ApplyPortMappingPolicy([]PortMapEntry{ + { + ContainerPort: 80, + HostPort: 8080, + Protocol: "TCP", + HostIP: "192.168.1.2", + }, + }) + + // only one item addlArgs := n.Policies Expect(addlArgs).Should(HaveLen(1)) + // normal type judgement policy := addlArgs[0] Expect(policy.Name).Should(Equal("EndpointPolicy")) - - var value map[string]interface{} + value := make(map[string]interface{}) json.Unmarshal(policy.Value, &value) - Expect(value).Should(HaveKey("Type")) - Expect(value).Should(HaveKey("ExceptionList")) - Expect(value["Type"]).Should(Equal("OutBoundNAT")) + Expect(value["Type"]).Should(Equal("PortMapping")) + Expect(value).Should(HaveKey("Settings")) - exceptionList := value["ExceptionList"].([]interface{}) - Expect(exceptionList).Should(HaveLen(2)) - Expect(exceptionList[0].(string)).Should(Equal("192.168.0.0/16")) - Expect(exceptionList[1].(string)).Should(Equal("10.244.0.0/16")) + // compare all values + settings := value["Settings"].(map[string]interface{}) + Expect(settings).Should(HaveKey("InternalPort")) + Expect(settings["InternalPort"]).Should(Equal(float64(80))) + Expect(settings).Should(HaveKey("ExternalPort")) + Expect(settings["ExternalPort"]).Should(Equal(float64(8080))) + Expect(settings).Should(HaveKey("Protocol")) + Expect(settings["Protocol"]).Should(Equal(float64(6))) + Expect(settings).Should(HaveKey("VIP")) + Expect(settings["VIP"]).Should(Equal("192.168.1.2")) + }) + }) + }) + + Describe("GetXEndpointPolicies", func() { + Context("via v1 api", func() { + var n NetConf + BeforeEach(func() { + n = NetConf{} + }) + + It("GetHNSEndpointPolicies", func() { + // mock different policies + n.Policies = []Policy{ + { + Name: "EndpointPolicy", + Value: []byte(`{"Type": "OutBoundNAT", "ExceptionList": [ "192.168.1.2" ]}`), + }, + { + Name: "someOtherType", + Value: []byte(`{"someOtherKey": "someOtherValue"}`), + }, + } + + // only one valid item + result := n.GetHNSEndpointPolicies() + Expect(len(result)).To(Equal(1)) + + // normal type judgement + policy := make(map[string]interface{}) + err := json.Unmarshal(result[0], &policy) + Expect(err).ToNot(HaveOccurred()) + Expect(policy).Should(HaveKey("Type")) + Expect(policy["Type"]).To(Equal("OutBoundNAT")) + Expect(policy).Should(HaveKey("ExceptionList")) + Expect(policy["ExceptionList"]).To(ContainElement("192.168.1.2")) + }) + }) + + Context("via v2 api", func() { + var n NetConf + BeforeEach(func() { + n = NetConf{ApiVersion: 2} + }) + + It("GetHostComputeEndpointPolicies", func() { + // mock different policies + n.Policies = []Policy{ + { + Name: "EndpointPolicy", + Value: []byte(`{"Type": "OutBoundNAT", "Settings": {"Exceptions": [ "192.168.1.2" ]}}`), + }, + { + Name: "someOtherType", + Value: []byte(`{"someOtherKey": "someOtherValue"}`), + }, + } + + // only one valid item + result := n.GetHostComputeEndpointPolicies() + Expect(len(result)).To(Equal(1)) + + // normal type judgement + policy := result[0] + Expect(policy.Type).Should(Equal(hcn.OutBoundNAT)) + settings := make(map[string]interface{}) + err := json.Unmarshal(policy.Settings, &settings) + Expect(err).ToNot(HaveOccurred()) + Expect(settings["Exceptions"]).To(ContainElement("192.168.1.2")) }) }) }) diff --git a/plugins/main/windows/win-bridge/sample.conf b/plugins/main/windows/win-bridge/sample-v1.conf similarity index 100% rename from plugins/main/windows/win-bridge/sample.conf rename to plugins/main/windows/win-bridge/sample-v1.conf diff --git a/plugins/main/windows/win-bridge/sample-v2.conf b/plugins/main/windows/win-bridge/sample-v2.conf new file mode 100644 index 00000000..3aabbda8 --- /dev/null +++ b/plugins/main/windows/win-bridge/sample-v2.conf @@ -0,0 +1,52 @@ +{ + "name":"cbr0", + "type":"flannel", + "delegate":{ + "apiVersion":2, + "type":"win-bridge", + "dns":{ + "nameservers":[ + "11.0.0.10" + ], + "search":[ + "svc.cluster.local" + ] + }, + "policies":[ + { + "name":"EndpointPolicy", + "value":{ + "Type":"OutBoundNAT", + "Settings":{ + "Exceptions":[ + "192.168.0.0/16", + "11.0.0.0/8", + "10.137.196.0/23" + ] + } + } + }, + { + "name":"EndpointPolicy", + "value":{ + "Type":"SDNRoute", + "Settings":{ + "DestinationPrefix":"11.0.0.0/8", + "NeedEncap":true + } + } + }, + { + "name":"EndpointPolicy", + "value":{ + "Type":"SDNRoute", + "Settings":{ + "DestinationPrefix":"10.137.198.27/32", + "NeedEncap":true + } + } + } + ], + "loopbackDSR":true + } +} \ No newline at end of file diff --git a/plugins/main/windows/win-bridge/win-bridge_windows.go b/plugins/main/windows/win-bridge/win-bridge_windows.go index 9ecf0456..0e2645fc 100644 --- a/plugins/main/windows/win-bridge/win-bridge_windows.go +++ b/plugins/main/windows/win-bridge/win-bridge_windows.go @@ -84,9 +84,7 @@ func processEndpointArgs(args *skel.CmdArgs, n *NetConf) (*hns.EndpointInfo, err } // configure sNAT exception - if len(n.IPMasqNetwork) != 0 { - n.ApplyOutboundNatPolicy(n.IPMasqNetwork) - } + n.ApplyOutboundNatPolicy(n.IPMasqNetwork) // add port mapping if any present n.ApplyPortMappingPolicy(n.RuntimeConfig.PortMaps) diff --git a/plugins/main/windows/win-overlay/win-overlay_windows.go b/plugins/main/windows/win-overlay/win-overlay_windows.go index b7e0325a..3882b9af 100644 --- a/plugins/main/windows/win-overlay/win-overlay_windows.go +++ b/plugins/main/windows/win-overlay/win-overlay_windows.go @@ -119,7 +119,7 @@ func cmdAdd(args *skel.CmdArgs) error { result.DNS = n.GetDNS() if n.LoopbackDSR { - n.ApplyLoopbackDSR(&ipAddr) + n.ApplyLoopbackDSRPolicy(&ipAddr) } hnsEndpoint := &hcsshim.HNSEndpoint{ Name: epName, @@ -129,7 +129,7 @@ func cmdAdd(args *skel.CmdArgs) error { GatewayAddress: gw, IPAddress: ipAddr, MacAddress: macAddr, - Policies: n.MarshalPolicies(), + Policies: n.GetHNSEndpointPolicies(), } return hnsEndpoint, nil