Merge pull request #87 from Charliekenney23/fix/externalip-kubeproxy-workaround

allow inter-service communication from within cluster via external LB

Author: 0xch4z

cloud/linode/loadbalancers.go

@@ -120,11 +120,13 @@ func (l *loadbalancers) getLatestServiceLoadBalancerStatus(ctx context.Context,
 
 // getNodeBalancerByStatus attempts to get the NodeBalancer from the IPv4 specified in the
 // most recent LoadBalancer status.
-func (l *loadbalancers) getNodeBalancerByStatus(ctx context.Context, service *v1.Service) (*linodego.NodeBalancer, error) {
+func (l *loadbalancers) getNodeBalancerByStatus(ctx context.Context, service *v1.Service) (nb *linodego.NodeBalancer, err error) {
 	for _, ingress := range service.Status.LoadBalancer.Ingress {
-		ipv4 := ingress.IP
-		if nb, err := l.getNodeBalancerByIPv4(ctx, service, ipv4); err == nil {
-			return nb, err
+		if ingress.Hostname != "" {
+			return l.getNodeBalancerByHostname(ctx, service, ingress.Hostname)
+		}
+		if ingress.IP != "" {
+			return l.getNodeBalancerByIPv4(ctx, service, ingress.IP)
 		}
 	}
 	return nil, lbNotFoundError{serviceNn: getServiceNn(service)}
@@ -436,6 +438,20 @@ func (l *loadbalancers) EnsureLoadBalancerDeleted(ctx context.Context, clusterNa
 	return nil
 }
 
+func (l *loadbalancers) getNodeBalancerByHostname(ctx context.Context, service *v1.Service, hostname string) (*linodego.NodeBalancer, error) {
+	lbs, err := l.client.ListNodeBalancers(ctx, nil)
+	if err != nil {
+		return nil, err
+	}
+	for _, lb := range lbs {
+		if *lb.Hostname == hostname {
+			klog.V(2).Infof("found NodeBalancer (%d) for service (%s) via hostname (%s)", lb.ID, getServiceNn(service), hostname)
+			return &lb, nil
+		}
+	}
+	return nil, lbNotFoundError{serviceNn: getServiceNn(service)}
+}
+
 func (l *loadbalancers) getNodeBalancerByIPv4(ctx context.Context, service *v1.Service, ipv4 string) (*linodego.NodeBalancer, error) {
 	lbs, err := l.client.ListNodeBalancers(ctx, nil)
 	if err != nil {
@@ -756,7 +772,6 @@ func getConnectionThrottle(service *v1.Service) int {
 func makeLoadBalancerStatus(nb *linodego.NodeBalancer) *v1.LoadBalancerStatus {
 	return &v1.LoadBalancerStatus{
 		Ingress: []v1.LoadBalancerIngress{{
-			IP:       *nb.IPv4,
 			Hostname: *nb.Hostname,
 		}},
 	}

cloud/linode/loadbalancers_test.go

@@ -310,9 +310,9 @@ func testUpdateLoadBalancerAddAnnotation(t *testing.T, client *linodego.Client,
 		t.Errorf("UpdateLoadBalancer returned an error while updated annotations: %s", err)
 	}
 
-	nb, err := lb.getNodeBalancerByIPv4(context.TODO(), svc, lbStatus.Ingress[0].IP)
+	nb, err := lb.getNodeBalancerByStatus(context.TODO(), svc)
 	if err != nil {
-		t.Errorf("unexpected error: %v", err)
+		t.Fatalf("failed to get NodeBalancer via status: %s", err)
 	}
 
 	if nb.ClientConnThrottle != 10 {

e2e/test/ccm_e2e_test.go

@@ -109,7 +109,7 @@ var _ = Describe("e2e tests", func() {
 	var getResponseFromSamePod = func(link string) {
 		var oldResp, newResp string
 		Eventually(func() string {
-			resp, err := http.Get(link)
+			resp, err := http.Get("http://" + link)
 			if err == nil {
 				byteData, _ := ioutil.ReadAll(resp.Body)
 				defer resp.Body.Close()

e2e/test/framework/loadbalancer_suite.go

@@ -15,7 +15,7 @@ func (i *lbInvocation) GetHTTPEndpoints() ([]string, error) {
 }
 
 func (i *lbInvocation) GetNodeBalancerID(svcName string) (int, error) {
-	ip, err := i.waitForLoadBalancerIP(svcName)
+	hostname, err := i.waitForLoadBalancerHostname(svcName)
 	if err != nil {
 		return -1, err
 	}
@@ -27,7 +27,7 @@ func (i *lbInvocation) GetNodeBalancerID(svcName string) (int, error) {
 	}
 
 	for _, nb := range nbList {
-		if *nb.IPv4 == ip {
+		if *nb.Hostname == hostname {
 			return nb.ID, nil
 		}
 	}
@@ -74,7 +74,7 @@ func (i *lbInvocation) GetNodeBalancerConfigForPort(svcName string, port int) (*
 	return nil, fmt.Errorf("NodeBalancerConfig for port %d was not found", port)
 }
 
-func (i *lbInvocation) waitForLoadBalancerIP(svcName string) (string, error) {
+func (i *lbInvocation) waitForLoadBalancerHostname(svcName string) (string, error) {
 	var ip string
 	err := wait.PollImmediate(RetryInterval, RetryTimeout, func() (bool, error) {
 		svc, err := i.kubeClient.CoreV1().Services(i.Namespace()).Get(context.TODO(), svcName, metav1.GetOptions{})
@@ -85,7 +85,7 @@ func (i *lbInvocation) waitForLoadBalancerIP(svcName string) (string, error) {
 		if svc.Status.LoadBalancer.Ingress == nil {
 			return false, nil
 		}
-		ip = svc.Status.LoadBalancer.Ingress[0].IP
+		ip = svc.Status.LoadBalancer.Ingress[0].Hostname
 		return true, nil
 	})
 

e2e/test/framework/service.go

@@ -5,7 +5,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"log"
-	"net/url"
 	"time"
 
 	"github.com/golang/glog"
@@ -149,9 +148,9 @@ func (i *lbInvocation) getLoadBalancerURLs() ([]string, error) {
 		return serverAddr, err
 	}
 
-	ips := make([]string, 0)
+	hostnames := make([]string, 0)
 	for _, ingress := range svc.Status.LoadBalancer.Ingress {
-		ips = append(ips, ingress.IP)
+		hostnames = append(hostnames, ingress.Hostname)
 	}
 
 	var ports []int32
@@ -164,12 +163,8 @@ func (i *lbInvocation) getLoadBalancerURLs() ([]string, error) {
 	}
 
 	for _, port := range ports {
-		for _, ip := range ips {
-			u, err := url.Parse(fmt.Sprintf("http://%s:%d", ip, port))
-			if err != nil {
-				return nil, err
-			}
-			serverAddr = append(serverAddr, u.String())
+		for _, hostname := range hostnames {
+			serverAddr = append(serverAddr, fmt.Sprintf("%s:%d", hostname, port))
 		}
 	}
 

e2e/test/framework/util.go

@@ -8,7 +8,6 @@ import (
 	"log"
 	"net"
 	"net/http"
-	"net/url"
 	"os"
 	"os/exec"
 	"path"
@@ -142,13 +141,10 @@ func getHTTPSResponse(domain, ip, port string) (string, error) {
 
 func WaitForHTTPSResponse(link string, podName string) error {
 	return wait.PollImmediate(RetryInterval, RetryTimeout, func() (bool, error) {
-		u, err := url.Parse(link)
-		if err != nil {
-			return false, nil
-		}
-		ipPort := strings.Split(u.Host, ":")
+		hostPort := strings.Split(link, ":")
+		host, port := hostPort[0], hostPort[1]
 
-		resp, err := getHTTPSResponse(Domain, ipPort[0], ipPort[1])
+		resp, err := getHTTPSResponse(Domain, host, port)
 		if err != nil {
 			return false, nil
 		}
@@ -163,7 +159,7 @@ func WaitForHTTPSResponse(link string, podName string) error {
 }
 
 func getHTTPResponse(link string) (bool, string, error) {
-	resp, err := http.Get(link)
+	resp, err := http.Get("http://" + link)
 	if err != nil {
 		return false, "", err
 	}
@@ -173,7 +169,6 @@ func getHTTPResponse(link string) (bool, string, error) {
 	if err != nil {
 		return false, "", err
 	}
-
 	return resp.StatusCode == 200, string(bodyBytes), nil
 }