Add ingress loadbalancer status for elb and Merlin (#174)

* Adding ingress status updater tools This is to add the underlying framework needed to add updater status updaters which will allow us to set the status of an ingress. Adding the ingress resource to the `IngressEntry` so that it can be used at a later stage. Also including useful util functions which can be reused by multiple status updaters. * Adding elb status updater This is to add a new status updater for use with the elb's. It will discover all of the elb's with the relevant labels (using existing function as part of the elb updater). After this, it will iterate through the ingress resources and set the relevent elb DNS name based on the lb scheme label. * Adding Merlin status updater This is to add a new status updater for use with Merlin. If enabled, the user will need to specify both `internal-hostname` and `external-hostname` as the updater uses these two values to match each ingress with the two hostnames or ip addresses using the lb scheme label. * Updating README and CHANGELOG This is to add the changes that have been made to update the ingress status to the README and create a new version in the CHANGELOG `v1.10.0`.
sky-uk · May 25, 2018 · 3723a28 · 3723a28
1 parent 24b40b5
commit 3723a28
Show file tree

Hide file tree

Showing 15 changed files with 601 additions and 48 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,7 @@
+# v1.10.0
+* Added `k8s/status` library for setting ingress status
+* ELB and Merlin updaters set relevant ingress status
+
 # v1.9.3
 * Attach to https in addition to http for merlin.
 * Fix merlin deregistration, which was failing due to long lived connections getting killed.

diff --git a/README.md b/README.md
@@ -89,6 +89,26 @@ securityContext:
 
 See the [example ingress for gorb](examples/feed-ingress-deployment-gorb.yml)
 
+### Ingress status
+
+When using either the [elb](#elb) or [Merlin](#merlin) updater, the ingress status will be updated with relevant
+loadbalancer information. This can then be used with other controllers such a `external-dns` which can set DNS for any
+given ingress using the ingress status.
+
+#### elb
+
+feed will automatically discover all of your elb's and then use the `sky.uk/frontend-scheme` annotation to match an elb
+label to an ingress. The updater will then set the ingress status to the elb's DNS name. 
+
+#### Merlin
+
+The Merlin updater is currently unable to auto-discover all hosted vips (virtual ip addresses) on a Merlin server;
+instead the status updater supports two different loadbalancer types: `internal` and `internet-facing`. These two vips
+are set using the `merlin-vip` and `merlin-internet-facing-vip` flags respectively.
+
+An ingress can select which loadbalancer it wants to be associated with by setting the `sky.uk/frontend-scheme`
+annotation to either `internal` or `internet-facing`.
+
 ### Running feed-ingress on privileged ports
 
 feed-ingress can be run on privileged ports by defining  the `NET_BIND_SERVICE` Linux capability.

diff --git a/cmd/feed-ingress/main.go b/cmd/feed-ingress/main.go
@@ -16,9 +16,11 @@ import (
 	"github.com/sky-uk/feed/alb"
 	"github.com/sky-uk/feed/controller"
 	"github.com/sky-uk/feed/elb"
+	elb_status "github.com/sky-uk/feed/elb/status"
 	"github.com/sky-uk/feed/gorb"
 	"github.com/sky-uk/feed/k8s"
 	"github.com/sky-uk/feed/merlin"
+	merlin_status "github.com/sky-uk/feed/merlin/status"
 	"github.com/sky-uk/feed/nginx"
 	"github.com/sky-uk/feed/util/cmd"
 	"github.com/sky-uk/feed/util/metrics"
@@ -72,6 +74,7 @@ var (
 	merlinHealthTimeout            time.Duration
 	merlinVIP                      string
 	merlinVIPInterface             string
+	merlinInternetFacingVIP        string
 )
 
 const (
@@ -276,6 +279,7 @@ func init() {
 	flag.StringVar(&merlinVIP, "merlin-vip", "", "VIP to assign to loopback to support direct route and tunnel.")
 	flag.StringVar(&merlinVIPInterface, "merlin-vip-interface", defaultMerlinVIPInterface,
 		"VIP interface to assign the VIP.")
+	flag.StringVar(&merlinInternetFacingVIP, "merlin-internet-facing-vip", "", "VIP to assign internet facing DNS.")
 }
 
 func main() {
@@ -290,7 +294,7 @@ func main() {
 	}
 	controllerConfig.KubernetesClient = client
 
-	controllerConfig.Updaters, err = createIngressUpdaters()
+	controllerConfig.Updaters, err = createIngressUpdaters(client)
 	if err != nil {
 		log.Fatal("Unable to create ingress updaters: ", err)
 	}
@@ -314,7 +318,7 @@ func main() {
 	select {}
 }
 
-func createIngressUpdaters() ([]controller.Updater, error) {
+func createIngressUpdaters(kubernetesClient k8s.Client) ([]controller.Updater, error) {
 	nginxConfig.Ports = []nginx.Port{{Name: "http", Port: ingressPort}}
 	if ingressHTTPSPort != unset {
 		nginxConfig.Ports = append(nginxConfig.Ports, nginx.Port{Name: "https", Port: ingressHTTPSPort})
@@ -338,6 +342,17 @@ func createIngressUpdaters() ([]controller.Updater, error) {
 		}
 		updaters = append(updaters, elbUpdater)
 
+		statusConfig := elb_status.Config{
+			Region:           region,
+			LabelValue:       elbLabelValue,
+			KubernetesClient: kubernetesClient,
+		}
+		elbStatusUpdater, err := elb_status.New(statusConfig)
+		if err != nil {
+			return updaters, err
+		}
+		updaters = append(updaters, elbStatusUpdater)
+
 	case "alb":
 		albUpdater, err := alb.New(region, targetGroupNames, targetGroupDeregistrationDelay)
 		if err != nil {
@@ -401,8 +416,20 @@ func createIngressUpdaters() ([]controller.Updater, error) {
 		}
 		updaters = append(updaters, merlinUpdater)
 
+		statusConfig := merlin_status.Config{
+			InternalVIP:       merlinVIP,
+			InternetFacingVIP: merlinInternetFacingVIP,
+			KubernetesClient:  kubernetesClient,
+		}
+		merlinStatusUpdater, err := merlin_status.New(statusConfig)
+		if err != nil {
+			return updaters, err
+		}
+		updaters = append(updaters, merlinStatusUpdater)
+
 	default:
-		return nil, fmt.Errorf("invalid registration frontend type. Must be either gorb, elb, alb but was %s", registrationFrontendType)
+		return nil, fmt.Errorf("invalid registration frontend type. Must be either gorb, elb, alb, merlin but"+
+			"was %s", registrationFrontendType)
 	}
 
 	return updaters, nil

diff --git a/controller/controller.go b/controller/controller.go
@@ -176,14 +176,15 @@ func (c *controller) updateIngresses() error {
 						BackendTimeoutSeconds: c.defaultBackendTimeout,
 						BackendMaxConnections: c.defaultBackendMaxConnections,
 						CreationTimestamp:     ingress.CreationTimestamp.Time,
+						Ingress:               ingress,
 					}
 
 					log.Debugf("Found ingress to update: %s", ingress.Name)
 
-					if elbScheme, ok := ingress.Annotations[frontendSchemeAnnotation]; ok {
-						entry.ELbScheme = elbScheme
+					if lbScheme, ok := ingress.Annotations[frontendSchemeAnnotation]; ok {
+						entry.LbScheme = lbScheme
 					} else {
-						entry.ELbScheme = ingress.Annotations[frontendElbSchemeAnnotation]
+						entry.LbScheme = ingress.Annotations[frontendElbSchemeAnnotation]
 					}
 
 					if allow, ok := ingress.Annotations[ingressAllowAnnotation]; ok {

diff --git a/controller/controller_test.go b/controller/controller_test.go
@@ -376,7 +376,7 @@ func TestUpdaterIsUpdatedOnK8sUpdates(t *testing.T) {
 				Path:           ingressPath,
 				ServiceAddress: serviceIP,
 				ServicePort:    ingressSvcPort,
-				ELbScheme:      "internal",
+				LbScheme:       "internal",
 				Allow:          []string{},
 				BackendTimeoutSeconds: backendTimeout,
 			}},
@@ -392,7 +392,7 @@ func TestUpdaterIsUpdatedOnK8sUpdates(t *testing.T) {
 				Path:                  ingressPath,
 				ServiceAddress:        serviceIP,
 				ServicePort:           ingressSvcPort,
-				ELbScheme:             "internal",
+				LbScheme:              "internal",
 				Allow:                 []string{},
 				StripPaths:            true,
 				BackendTimeoutSeconds: backendTimeout,
@@ -409,7 +409,7 @@ func TestUpdaterIsUpdatedOnK8sUpdates(t *testing.T) {
 				Path:                  ingressPath,
 				ServiceAddress:        serviceIP,
 				ServicePort:           ingressSvcPort,
-				ELbScheme:             "internal",
+				LbScheme:              "internal",
 				Allow:                 []string{},
 				StripPaths:            false,
 				BackendTimeoutSeconds: backendTimeout,
@@ -426,7 +426,7 @@ func TestUpdaterIsUpdatedOnK8sUpdates(t *testing.T) {
 				Path:                  ingressPath,
 				ServiceAddress:        serviceIP,
 				ServicePort:           ingressSvcPort,
-				ELbScheme:             "internal",
+				LbScheme:              "internal",
 				Allow:                 []string{},
 				StripPaths:            false,
 				BackendTimeoutSeconds: 20,
@@ -443,7 +443,7 @@ func TestUpdaterIsUpdatedOnK8sUpdates(t *testing.T) {
 				Path:                  ingressPath,
 				ServiceAddress:        serviceIP,
 				ServicePort:           ingressSvcPort,
-				ELbScheme:             "internal",
+				LbScheme:              "internal",
 				Allow:                 []string{},
 				StripPaths:            false,
 				BackendTimeoutSeconds: backendTimeout,
@@ -460,7 +460,7 @@ func TestUpdaterIsUpdatedOnK8sUpdates(t *testing.T) {
 				Path:                  ingressPath,
 				ServiceAddress:        serviceIP,
 				ServicePort:           ingressSvcPort,
-				ELbScheme:             "internal",
+				LbScheme:              "internal",
 				Allow:                 []string{},
 				StripPaths:            false,
 				BackendTimeoutSeconds: 20,
@@ -478,7 +478,7 @@ func TestUpdaterIsUpdatedOnK8sUpdates(t *testing.T) {
 				Path:                  ingressPath,
 				ServiceAddress:        serviceIP,
 				ServicePort:           ingressSvcPort,
-				ELbScheme:             "internal",
+				LbScheme:              "internal",
 				Allow:                 []string{},
 				StripPaths:            false,
 				BackendTimeoutSeconds: 20,
@@ -489,6 +489,10 @@ func TestUpdaterIsUpdatedOnK8sUpdates(t *testing.T) {
 
 	for _, test := range tests {
 		fmt.Printf("test: %s\n", test.description)
+		// add ingress pointers to entries
+		test.entries = addIngresses(test.ingresses, test.entries)
+
+		// setup clients
 		client := new(fake.FakeClient)
 		updater := new(fakeUpdater)
 		controller := newController(updater, client)
@@ -519,6 +523,18 @@ func TestUpdaterIsUpdatedOnK8sUpdates(t *testing.T) {
 	}
 }
 
+func addIngresses(ingresses []*v1beta1.Ingress, entries IngressEntries) IngressEntries {
+	if len(ingresses) != len(entries) {
+		return entries
+	}
+	appendedEntries := IngressEntries{}
+	for i, entry := range entries {
+		entry.Ingress = ingresses[i]
+		appendedEntries = append(appendedEntries, entry)
+	}
+	return appendedEntries
+}
+
 func createLbEntriesFixture() IngressEntries {
 	return []IngressEntry{{
 		Namespace:             ingressNamespace,
@@ -528,7 +544,7 @@ func createLbEntriesFixture() IngressEntries {
 		ServiceAddress:        serviceIP,
 		ServicePort:           ingressSvcPort,
 		Allow:                 strings.Split(ingressAllow, ","),
-		ELbScheme:             elbScheme,
+		LbScheme:              lbScheme,
 		BackendTimeoutSeconds: backendTimeout,
 	}}
 }
@@ -543,7 +559,7 @@ const (
 	ingressAllow          = "10.82.0.0/16,10.44.0.0/16"
 	ingressDefaultAllow   = "10.50.0.0/16,10.1.0.0/16"
 	serviceIP             = "10.254.0.82"
-	elbScheme             = "internal"
+	lbScheme              = "internal"
 	stripPath             = "MISSING"
 	backendTimeout        = 10
 	defaultMaxConnections = 0
@@ -571,7 +587,7 @@ func createIngressesFixture(host string, serviceName string, servicePort int, al
 	annotations := make(map[string]string)
 	if allow != "MISSING" {
 		annotations[ingressAllowAnnotation] = allow
-		annotations[schemeAnnotationKey] = elbScheme
+		annotations[schemeAnnotationKey] = lbScheme
 	}
 	if stripPath != "MISSING" {
 		annotations[stripPathAnnotation] = stripPath

diff --git a/controller/ingress_entry.go b/controller/ingress_entry.go
@@ -4,6 +4,8 @@ import (
 	"errors"
 	"fmt"
 	"time"
+
+	"k8s.io/client-go/pkg/apis/extensions/v1beta1"
 )
 
 // IngressEntries type
@@ -26,8 +28,8 @@ type IngressEntry struct {
 	ServicePort int32
 	// Allow are the ips or cidrs that are allowed to access the service.
 	Allow []string
-	// ElbScheme internet-facing or internal will dictate which kind of ELB to attach to
-	ELbScheme string
+	// LbScheme internet-facing or internal will dictate which kind of load balancer to attach to.
+	LbScheme string
 	// StripPaths before forwarding to the backend
 	StripPaths bool
 	// BackendTimeoutSeconds backend timeout
@@ -36,6 +38,8 @@ type IngressEntry struct {
 	BackendMaxConnections int
 	// Ingress creation time
 	CreationTimestamp time.Time
+	// Ingress resource
+	Ingress *v1beta1.Ingress
 }
 
 // validate returns error if entry has invalid fields.

diff --git a/dns/dns_updater.go b/dns/dns_updater.go
@@ -164,8 +164,8 @@ func (u *updater) indexByHost(entries []controller.IngressEntry) (hostToIngress,
 		}
 
 		if previous, exists := mapping[hostNameWithPeriod]; exists {
-			if previous.ELbScheme != entry.ELbScheme {
-				skipped = append(skipped, entry.NamespaceName()+":conflicting-scheme:"+entry.ELbScheme)
+			if previous.LbScheme != entry.LbScheme {
+				skipped = append(skipped, entry.NamespaceName()+":conflicting-scheme:"+entry.LbScheme)
 				skippedCount.Inc()
 			}
 		} else {
@@ -188,9 +188,9 @@ func (u *updater) createChanges(hostToIngress hostToIngress,
 
 	var skipped []string
 	for host, entry := range hostToIngress {
-		dnsDetails, exists := u.schemeToFrontendMap[entry.ELbScheme]
+		dnsDetails, exists := u.schemeToFrontendMap[entry.LbScheme]
 		if !exists {
-			skipped = append(skipped, entry.NamespaceName()+":scheme:"+entry.ELbScheme)
+			skipped = append(skipped, entry.NamespaceName()+":scheme:"+entry.LbScheme)
 			skippedCount.Inc()
 			continue
 		}