diff --git a/README.md b/README.md index 6a8e78a8..f9cc57dc 100644 --- a/README.md +++ b/README.md @@ -44,8 +44,8 @@ Annotation (Suffix) | Values | Default | Description ---|---|---|--- `throttle` | `0`-`20` (`0` to disable) | `20` | Client Connection Throttle, which limits the number of subsequent new connections per second from the same client IP `default-protocol` | `tcp`, `http`, `https` | `tcp` | This annotation is used to specify the default protocol for Linode NodeBalancer. -`proxy-protocol` | `none`, `v1`, `v2` | `none` | Specifies whether to use a version of Proxy Protocol on the underlying NodeBalancer -`port-*` | json (e.g. `{ "tls-secret-name": "prod-app-tls", "protocol": "https"}`) | | Specifies the secret and protocol for a port corresponding secrets. The secret type should be `kubernetes.io/tls`. `*` is the port being configured, e.g. `linode-loadbalancer-port-443` +`default-proxy-protocol` | `none`, `v1`, `v2` | `none` | Specifies whether to use a version of Proxy Protocol on the underlying NodeBalancer. +`port-*` | json (e.g. `{ "tls-secret-name": "prod-app-tls", "protocol": "https", "proxy-protocol": "v2"}`) | | Specifies port specific NodeBalancer configuration. See [Port Specific Configuration](#port-specific-configuration). `*` is the port being configured, e.g. `linode-loadbalancer-port-443` `check-type` | `none`, `connection`, `http`, `http_body` | | The type of health check to perform against back-ends to ensure they are serving requests `check-path` | string | | The URL path to check on each back-end during health checks `check-body` | string | | Text which must be present in the response body to pass the NodeBalancer health check @@ -58,17 +58,28 @@ Annotation (Suffix) | Values | Default | Description #### Deprecated Annotations -These annotations are deprecated, and will be removed Q3 2020. +These annotations are deprecated, and will be removed in a future release. -Annotation (Suffix) | Values | Default | Description ----|---|---|--- -`protocol` | `tcp`, `http`, `https` | `tcp` | This annotation is used to specify the default protocol for Linode NodeBalancer. For ports specified in the `linode-loadbalancer-tls-ports` annotation, this protocol is overwritten to `https` -`tls` | json array (e.g. `[ { "tls-secret-name": "prod-app-tls", "port": 443}, {"tls-secret-name": "dev-app-tls", "port": 8443} ]`) | | Specifies TLS ports with their corresponding secrets, the secret type should be `kubernetes.io/tls +Annotation (Suffix) | Values | Default | Description | Scheduled Removal +---|---|---|---|--- +`protocol` | `tcp`, `http`, `https` | `tcp` | This annotation is used to specify the default protocol for Linode NodeBalancer. For ports specified in the `linode-loadbalancer-tls-ports` annotation, this protocol is overwritten to `https` | Q4 2020 +`proxy-protcol` | `none`, `v1`, `v2` | `none` | Specifies whether to use a version of Proxy Protocol on the underlying NodeBalancer | Q4 2021 +`tls` | json array (e.g. `[ { "tls-secret-name": "prod-app-tls", "port": 443}, {"tls-secret-name": "dev-app-tls", "port": 8443} ]`) | | Specifies TLS ports with their corresponding secrets, the secret type should be `kubernetes.io/tls | Q4 2020 #### Annotation bool values For annotations with bool value types, `"1"`, `"t"`, `"T"`, `"True"`, `"true"` and `"True"` are valid string representations of `true`. Any other values will be interpreted as false. For more details, see [strconv.ParseBool](https://golang.org/pkg/strconv/#ParseBool). +#### Port Specific Configuration + +These configuration options can be specified via the `port-*` annotation, encoded in JSON. + +Key | Values | Default | Description +---|---|---|--- +`protocol` | `tcp`, `http`, `https` | `tcp` | Specifies protocol of the NodeBalancer port. Overwrites `default-protocol`. +`proxy-protocol` | `none`, `v1`, `v2` | `none` | Specifies whether to use a version of Proxy Protocol on the underlying NodeBalancer. Overwrites `default-proxy-protocol`. +`tls-secret-name` | string | | Specifies a secret to use for TLS. The secret type should be `kubernetes.io/tls`. + #### Example usage ```yaml @@ -128,10 +139,10 @@ spec: See more in the [examples directory](examples) -## Why `stickiness` and `algorithm` annotations don't exit +## Why `stickiness` and `algorithm` annotations don't exist As kube-proxy will simply double-hop the traffic to a random backend Pod anyway, it doesn't matter which backend Node traffic is forwarded-to for the sake of session stickiness. -So these annotations are not necessary to implement session stickiness. +These annotations are not necessary to implement session stickiness, as kube-proxy will simply double-hop the packets to a random backend Pod. It would not make a difference to set a backend Node that would receive the network traffic in an attempt to set session stickiness. ## How to use sessionAffinity diff --git a/cloud/linode/loadbalancers.go b/cloud/linode/loadbalancers.go index 90570778..6afe6a08 100644 --- a/cloud/linode/loadbalancers.go +++ b/cloud/linode/loadbalancers.go @@ -24,9 +24,9 @@ import ( const ( // annLinodeDefaultProtocol is the annotation used to specify the default protocol // for Linode load balancers. Options are tcp, http and https. Defaults to tcp. - annLinodeDefaultProtocol = "service.beta.kubernetes.io/linode-loadbalancer-default-protocol" - annLinodePortConfigPrefix = "service.beta.kubernetes.io/linode-loadbalancer-port-" - annLinodeProxyProtocol = "service.beta.kubernetes.io/linode-loadbalancer-proxy-protocol" + annLinodeDefaultProtocol = "service.beta.kubernetes.io/linode-loadbalancer-default-protocol" + annLinodePortConfigPrefix = "service.beta.kubernetes.io/linode-loadbalancer-port-" + annLinodeDefaultProxyProtocol = "service.beta.kubernetes.io/linode-loadbalancer-default-proxy-protocol" annLinodeCheckPath = "service.beta.kubernetes.io/linode-loadbalancer-check-path" annLinodeCheckBody = "service.beta.kubernetes.io/linode-loadbalancer-check-body" @@ -68,11 +68,13 @@ type loadbalancers struct { type portConfigAnnotation struct { TLSSecretName string `json:"tls-secret-name"` Protocol string `json:"protocol"` + ProxyProtocol string `json:"proxy-protocol"` } type portConfig struct { TLSSecretName string Protocol linodego.ConfigProtocol + ProxyProtocol linodego.ConfigProxyProtocol Port int } @@ -478,9 +480,10 @@ func (l *loadbalancers) buildNodeBalancerConfig(service *v1.Service, port int) ( } config := linodego.NodeBalancerConfig{ - Port: port, - Protocol: portConfig.Protocol, - Check: health, + Port: port, + Protocol: portConfig.Protocol, + ProxyProtocol: portConfig.ProxyProtocol, + Check: health, } if health == linodego.CheckHTTP || health == linodego.CheckHTTPBody { @@ -530,17 +533,6 @@ func (l *loadbalancers) buildNodeBalancerConfig(service *v1.Service, port int) ( } config.CheckPassive = checkPassive - proxyProtocol := linodego.ProxyProtocolNone - if pp, ok := service.Annotations[annLinodeProxyProtocol]; ok { - switch linodego.ConfigProxyProtocol(pp) { - case linodego.ProxyProtocolNone, linodego.ProxyProtocolV1, linodego.ProxyProtocolV2: - proxyProtocol = linodego.ConfigProxyProtocol(pp) - default: - return config, fmt.Errorf("invalid NodeBalancer proxy protocol value '%s'", pp) - } - } - config.ProxyProtocol = proxyProtocol - if portConfig.Protocol == linodego.ProtocolHTTPS { if err = l.addTLSCert(service, &config, portConfig); err != nil { return config, err @@ -643,15 +635,35 @@ func getPortConfig(service *v1.Service, port int) (portConfig, error) { protocol = "tcp" } } - protocol = strings.ToLower(protocol) + proxyProtocol := portConfigAnnotation.ProxyProtocol + if proxyProtocol == "" { + var ok bool + for _, ann := range []string{annLinodeDefaultProxyProtocol, annLinodeProxyProtocol} { + proxyProtocol, ok = service.Annotations[ann] + if ok { + break + } else { + proxyProtocol = string(linodego.ProxyProtocolNone) + } + } + } + if protocol != "tcp" && protocol != "http" && protocol != "https" { return portConfig, fmt.Errorf("invalid protocol: %q specified", protocol) } + switch proxyProtocol { + case string(linodego.ProxyProtocolNone), string(linodego.ProxyProtocolV1), string(linodego.ProxyProtocolV2): + break + default: + return portConfig, fmt.Errorf("invalid NodeBalancer proxy protocol value '%s'", proxyProtocol) + } + portConfig.Port = port portConfig.Protocol = linodego.ConfigProtocol(protocol) + portConfig.ProxyProtocol = linodego.ConfigProxyProtocol(proxyProtocol) portConfig.TLSSecretName = portConfigAnnotation.TLSSecretName return portConfig, nil diff --git a/cloud/linode/loadbalancers_deprecated.go b/cloud/linode/loadbalancers_deprecated.go index 2cd4ca75..c9047aa0 100644 --- a/cloud/linode/loadbalancers_deprecated.go +++ b/cloud/linode/loadbalancers_deprecated.go @@ -9,6 +9,7 @@ import ( const ( annLinodeProtocolDeprecated = "service.beta.kubernetes.io/linode-loadbalancer-protocol" annLinodeLoadBalancerTLSDeprecated = "service.beta.kubernetes.io/linode-loadbalancer-tls" + annLinodeProxyProtocol = "service.beta.kubernetes.io/linode-loadbalancer-proxy-protocol" ) type tlsAnnotationDeprecated struct { diff --git a/cloud/linode/loadbalancers_test.go b/cloud/linode/loadbalancers_test.go index 75f76b42..11bfb133 100644 --- a/cloud/linode/loadbalancers_test.go +++ b/cloud/linode/loadbalancers_test.go @@ -573,7 +573,7 @@ func testUpdateLoadBalancerAddProxyProtocol(t *testing.T, client *linodego.Clien svc.Status.LoadBalancer = *makeLoadBalancerStatus(nodeBalancer) svc.ObjectMeta.SetAnnotations(map[string]string{ - annLinodeProxyProtocol: string(tc.proxyProtocolConfig), + annLinodeDefaultProxyProtocol: string(tc.proxyProtocolConfig), }) stubService(fakeClientset, svc) @@ -771,6 +771,60 @@ func Test_getPortConfig(t *testing.T) { expectedPortConfig portConfig err error }{ + { + "default no proxy protocol specified", + &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: randString(10), + UID: "abc123", + }, + }, + portConfig{Port: 443, Protocol: "tcp", ProxyProtocol: linodego.ProxyProtocolNone}, + nil, + }, + { + "default proxy protocol specified", + &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: randString(10), + UID: "abc123", + Annotations: map[string]string{ + annLinodeDefaultProxyProtocol: string(linodego.ProxyProtocolV2), + }, + }, + }, + portConfig{Port: 443, Protocol: "tcp", ProxyProtocol: linodego.ProxyProtocolV2}, + nil, + }, + { + "port specific proxy protocol specified", + &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: randString(10), + UID: "abc123", + Annotations: map[string]string{ + annLinodeDefaultProxyProtocol: string(linodego.ProxyProtocolV2), + annLinodePortConfigPrefix + "443": fmt.Sprintf(`{"proxy-protocol": "%s"}`, linodego.ProxyProtocolV1), + }, + }, + }, + portConfig{Port: 443, Protocol: "tcp", ProxyProtocol: linodego.ProxyProtocolV1}, + nil, + }, + { + "default invalid proxy protocol", + &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: randString(10), + UID: "abc123", + Annotations: map[string]string{ + annLinodeDefaultProxyProtocol: "invalid", + }, + }, + }, + portConfig{}, + fmt.Errorf("invalid NodeBalancer proxy protocol value '%s'", "invalid"), + }, { "default no protocol specified", &v1.Service{ @@ -779,7 +833,7 @@ func Test_getPortConfig(t *testing.T) { UID: "abc123", }, }, - portConfig{Port: 443, Protocol: "tcp"}, + portConfig{Port: 443, Protocol: "tcp", ProxyProtocol: linodego.ProxyProtocolNone}, nil, }, @@ -794,7 +848,7 @@ func Test_getPortConfig(t *testing.T) { }, }, }, - portConfig{Port: 443, Protocol: "tcp"}, + portConfig{Port: 443, Protocol: "tcp", ProxyProtocol: linodego.ProxyProtocolNone}, nil, }, { @@ -808,7 +862,7 @@ func Test_getPortConfig(t *testing.T) { }, }, }, - portConfig{Port: 443, Protocol: "http"}, + portConfig{Port: 443, Protocol: "http", ProxyProtocol: linodego.ProxyProtocolNone}, nil, }, { @@ -837,7 +891,7 @@ func Test_getPortConfig(t *testing.T) { }, }, }, - portConfig{Port: 443, Protocol: "http"}, + portConfig{Port: 443, Protocol: "http", ProxyProtocol: linodego.ProxyProtocolNone}, nil, }, { @@ -851,7 +905,7 @@ func Test_getPortConfig(t *testing.T) { }, }, }, - portConfig{Port: 443, Protocol: "http"}, + portConfig{Port: 443, Protocol: "http", ProxyProtocol: linodego.ProxyProtocolNone}, nil, }, { diff --git a/e2e/test/ccm_e2e_test.go b/e2e/test/ccm_e2e_test.go index c96b0eb7..a8a25e18 100644 --- a/e2e/test/ccm_e2e_test.go +++ b/e2e/test/ccm_e2e_test.go @@ -3,6 +3,7 @@ package test import ( "context" "e2e_test/test/framework" + "fmt" "io/ioutil" "log" "net/http" @@ -27,6 +28,7 @@ var _ = Describe("e2e tests", func() { const ( annLinodeProxyProtocol = "service.beta.kubernetes.io/linode-loadbalancer-proxy-protocol" + annLinodeDefaultProxyProtocol = "service.beta.kubernetes.io/linode-loadbalancer-default-proxy-protocol" annLinodeDefaultProtocol = "service.beta.kubernetes.io/linode-loadbalancer-default-protocol" annLinodePortConfigPrefix = "service.beta.kubernetes.io/linode-loadbalancer-port-" annLinodeLoadBalancerPreserve = "service.beta.kubernetes.io/linode-loadbalancer-preserve" @@ -172,9 +174,9 @@ var _ = Describe("e2e tests", func() { Expect(err).NotTo(HaveOccurred()) } - var checkNodeBalancerConfig = func(args checkArgs) { - By("Getting NodeBalancer Configuration") - nbConfig, err := f.LoadBalancer.GetNodeBalancerConfig(framework.TestServerResourceName) + var checkNodeBalancerConfigForPort = func(port int, args checkArgs) { + By("Getting NodeBalancer Configuration for port " + strconv.Itoa(port)) + nbConfig, err := f.LoadBalancer.GetNodeBalancerConfigForPort(framework.TestServerResourceName, port) Expect(err).NotTo(HaveOccurred()) if args.checkType != "" { @@ -410,13 +412,19 @@ var _ = Describe("e2e tests", func() { labels map[string]string servicePorts []core.ServicePort - annotations = map[string]string{} + proxyProtocolV1 = string(linodego.ProxyProtocolV1) + proxyProtocolV2 = string(linodego.ProxyProtocolV2) + proxyProtocolNone = string(linodego.ProxyProtocolNone) ) BeforeEach(func() { pods = []string{"test-pod-1"} ports := []core.ContainerPort{ { Name: "http-1", + ContainerPort: 80, + }, + { + Name: "http-2", ContainerPort: 8080, }, } @@ -424,6 +432,12 @@ var _ = Describe("e2e tests", func() { { Name: "http-1", Port: 80, + TargetPort: intstr.FromInt(80), + Protocol: "TCP", + }, + { + Name: "http-2", + Port: 8080, TargetPort: intstr.FromInt(8080), Protocol: "TCP", }, @@ -437,7 +451,7 @@ var _ = Describe("e2e tests", func() { createPodWithLabel(pods, ports, framework.TestServerImage, labels, false) By("Creating Service") - createServiceWithAnnotations(labels, annotations, servicePorts, false) + createServiceWithAnnotations(labels, map[string]string{}, servicePorts, false) }) AfterEach(func() { @@ -448,15 +462,67 @@ var _ = Describe("e2e tests", func() { deleteService() }) - It("should update the NodeBalancer to use ProxyProtocol v2", func() { - proxyProtocolV2 := string(linodego.ProxyProtocolV2) + It("can set proxy-protocol on each port", func() { + By("Annotating port 80 with v1 and 8080 with v2") + updateServiceWithAnnotations(labels, map[string]string{ + annLinodePortConfigPrefix + "80": fmt.Sprintf(`{"proxy-protocol": "%s"}`, proxyProtocolV1), + annLinodePortConfigPrefix + "8080": fmt.Sprintf(`{"proxy-protocol": "%s"}`, proxyProtocolV2), + }, servicePorts, false) + + By("Checking NodeBalancerConfig for port 80 should have ProxyProtocol v1") + checkNodeBalancerConfigForPort(80, checkArgs{proxyProtocol: proxyProtocolV1}) + + By("Checking NodeBalancerConfig for port 8080 should have ProxyProtocol v2") + checkNodeBalancerConfigForPort(8080, checkArgs{proxyProtocol: proxyProtocolV2}) + }) + + It("should override default proxy-protocol annotation when a port configuration is specified", func() { + By("Annotating a default version of ProxyProtocol v2 and v1 for port 8080") + updateServiceWithAnnotations(labels, map[string]string{ + annLinodeDefaultProxyProtocol: proxyProtocolV2, + annLinodePortConfigPrefix + "8080": fmt.Sprintf(`{"proxy-protocol": "%s"}`, proxyProtocolV1), + }, servicePorts, false) + + By("Checking NodeBalancerConfig for port 80 should have the default ProxyProtocol v2") + checkNodeBalancerConfigForPort(80, checkArgs{proxyProtocol: proxyProtocolV2}) + + By("Checking NodeBalancerConfig for port 8080 should have ProxyProtocol v1") + checkNodeBalancerConfigForPort(8080, checkArgs{proxyProtocol: proxyProtocolV1}) + }) + + It("port specific configuration should not effect other ports", func() { + By("Annotating ProxyProtocol v2 on port 8080") + updateServiceWithAnnotations(labels, map[string]string{ + annLinodePortConfigPrefix + "8080": fmt.Sprintf(`{"proxy-protocol": "%s"}`, proxyProtocolV2), + }, servicePorts, false) + + By("Checking NodeBalancerConfig for port 8080 should have ProxyProtocolv2") + checkNodeBalancerConfigForPort(8080, checkArgs{proxyProtocol: proxyProtocolV2}) - By("Annotating ProxyProtocol v2") + By("Checking NodeBalancerConfig for port 80 should not have ProxyProtocol enabled") + checkNodeBalancerConfigForPort(80, checkArgs{proxyProtocol: proxyProtocolNone}) + }) + + It("default annotations can be used to apply ProxyProtocol to all NodeBalancerConfigs", func() { + annotations := make(map[string]string) + + By("By specifying ProxyProtocol v2 using the deprecated annotation " + annLinodeProxyProtocol) annotations[annLinodeProxyProtocol] = proxyProtocolV2 updateServiceWithAnnotations(labels, annotations, servicePorts, false) - By("Checking NodeBalancerConfig") - checkNodeBalancerConfig(checkArgs{proxyProtocol: proxyProtocolV2}) + By("Checking NodeBalancerConfig for port 80 should have default ProxyProtocol v2") + checkNodeBalancerConfigForPort(80, checkArgs{proxyProtocol: proxyProtocolV2}) + By("Checking NodeBalancerConfig for port 8080 should have ProxyProtocol v2") + checkNodeBalancerConfigForPort(8080, checkArgs{proxyProtocol: proxyProtocolV2}) + + By("specifying ProxyProtocol v1 using the annotation " + annLinodeDefaultProtocol) + annotations[annLinodeDefaultProxyProtocol] = proxyProtocolV1 + updateServiceWithAnnotations(labels, annotations, servicePorts, false) + + By("Checking NodeBalancerConfig for port 80 should have default ProxyProtocol v1") + checkNodeBalancerConfigForPort(80, checkArgs{proxyProtocol: proxyProtocolV1}) + By("Checking NodeBalancerConfig for port 8080 should have ProxyProtocol v1") + checkNodeBalancerConfigForPort(8080, checkArgs{proxyProtocol: proxyProtocolV1}) }) }) @@ -767,7 +833,7 @@ var _ = Describe("e2e tests", func() { It("should successfully check the health of 2 nodes", func() { By("Checking NodeBalancer Configurations") - checkNodeBalancerConfig(checkArgs{ + checkNodeBalancerConfigForPort(80, checkArgs{ checkType: checkType, path: path, body: body, @@ -1119,7 +1185,7 @@ var _ = Describe("e2e tests", func() { It("should successfully check the health of 2 nodes", func() { By("Checking NodeBalancer Configurations") - checkNodeBalancerConfig(checkArgs{ + checkNodeBalancerConfigForPort(80, checkArgs{ checkType: checkType, interval: interval, timeout: timeout, @@ -1181,7 +1247,7 @@ var _ = Describe("e2e tests", func() { It("should successfully check the health of 2 nodes", func() { By("Checking NodeBalancer Configurations") - checkNodeBalancerConfig(checkArgs{ + checkNodeBalancerConfigForPort(80, checkArgs{ checkType: checkType, checkPassive: checkPassive, checkNodes: true, @@ -1241,7 +1307,7 @@ var _ = Describe("e2e tests", func() { It("should successfully check the health of 2 nodes", func() { By("Checking NodeBalancer Configurations") - checkNodeBalancerConfig(checkArgs{ + checkNodeBalancerConfigForPort(80, checkArgs{ checkType: checkType, path: path, checkNodes: true, diff --git a/e2e/test/framework/loadbalancer_suite.go b/e2e/test/framework/loadbalancer_suite.go index e63f463b..63e33513 100644 --- a/e2e/test/framework/loadbalancer_suite.go +++ b/e2e/test/framework/loadbalancer_suite.go @@ -56,6 +56,24 @@ func (i *lbInvocation) GetNodeBalancerConfig(svcName string) (*linodego.NodeBala return &nbcList[0], nil } +func (i *lbInvocation) GetNodeBalancerConfigForPort(svcName string, port int) (*linodego.NodeBalancerConfig, error) { + id, err := i.GetNodeBalancerID(svcName) + if err != nil { + return nil, err + } + nbConfigs, err := i.linodeClient.ListNodeBalancerConfigs(context.Background(), id, nil) + if err != nil { + return nil, err + } + + for _, config := range nbConfigs { + if config.Port == port { + return &config, nil + } + } + return nil, fmt.Errorf("NodeBalancerConfig for port %d was not found", port) +} + func (i *lbInvocation) waitForLoadBalancerIP(svcName string) (string, error) { var ip string err := wait.PollImmediate(RetryInterval, RetryTimeout, func() (bool, error) {