Merge branch 'feat/ingressv1-informers-store' into next

Ingress V1: apiversion negotiation and flags, support in store,
parser-k8s decoupling, ingress parsing refactor

closes Kong#788

This PR does the following (each high-level bullet point roughly
matching a group of commits):
- refactors `parseIngressRules` into separate functions (without side
effects) and places them in `translate.go`: `fromIngressV1beta1` and
`fromTCPIngressV1beta1`. Also moves `parseKnativeIngressRules` to follow
suit. All of these get merged in the end using `mergeIngressRules`
recently implemented in Kong#822.
    - this comes with a tiny, potentially behavior changing, logic
    modification: after this PR, default backends get added to the
    overall set of `ingressRules` at a different stage of _"parsing"_
    than they were before.
    - this change is here as a preparation for introduction of
    `fromIngressV1` which will be a copy-paste of `fromIngressV1beta1`
    with changes as described in the ingress v1 KEP.
- removes concrete k8s types (`<your-favorite-apiversion>.Ingress`) and
`TCPIngress` from `package kongstate` by introducing an intermediate
`utils.K8sObjectInfo`:
    - I considered using k8s `ObjectMeta` instead of implementing
    `K8sObjectInfo` but that would violate separation between Kong and
    Kubernetes data formats that I wanted to achieve in `package
    kongstate`.
    - this change is here to accommodate Ingress V1.
- reimplements Ingress API version negotiation:
    - to look not only at supported API groups, but also at the specific
    resource kind available in that API group (because
    `networking.k8s.io/v1` existed in k8s before the `Ingress` kind was
    added thereto)
    - to support `networking.k8s.io/v1` alongside the already supported
    versions
    - to support enabling and disabling support for specific APIs in
    Kong. This is for several reasons:
	- this PR does not implement Ingress V1 fully, so we can merge
	this PR in with ingress v1 code paths disabled
	- if Ingress V1 support is enabled in KIC, and the apiserver is
	>=1.19, then KIC will use a completely new code path in parser
	**for all preexisting Ingress resources**, posing a significant
	regression risk for existing users. There are two possible
	strategies for Ingress v1 rollout:
	    - defensive: choose to leave ingress V1 APIs disabled by
	    default
	    - less defensive but with a workaround by design: make
	    ingress V1 APIs enabled by default, but have the disabling
	    flag readily at hand as a workaround for users whom we break
    - drive-by side effect: makes it possible to "soft-disable"
    extensions/v1beta1 ingress by setting its flag default to false
- implements Ingress v1 alongside v1beta1 in store.
    - Note that the distinction between v1 and both v1beta1 types is
    implemented differently than between `extensions` and `networking`
    v1beta1.
	- This is because `extensions` and `networking` v1beta1
	Ingresses are fully compatible, and they reuse the same parsing
	logic. Ingress V1, however, will be a separate implementation of
	the parsing logic. I wanted to ensure type safety (between
	v1beta1 and v1 ingresses), which would be impossible without
	differentiated storage.
- additionally: adds `*.orig` to `.gitignore`. `*.orig` files originate
from manually resolved Git conflicts and are easy to accidentally commit
into the repo.

Notes to the reviewer:
- Consider reviewing commit by commit.
- Commits are (intended to be) self-contained. Consider not squashing
when merging.

From Kong#826

Author: mflendrich

.gitignore

@@ -1,3 +1,6 @@
+# Git merge residuals
+*.orig
+
 # Compiled Object files, Static and Dynamic libs (Shared Objects)
 *.o
 *.a

cli/ingress-controller/flag_test.go

@@ -182,6 +182,10 @@ func TestOverrideViaCLIFlags(t *testing.T) {
 		"--apiserver-host", "kube-apiserver.internal",
 		"--kubeconfig", "/path/to/kubeconfig",
 
+		"--disable-ingress-extensionsv1beta1",
+		"--disable-ingress-networkingv1beta1",
+		"--disable-ingress-networkingv1",
+
 		"--log-format", "json",
 
 		"--profiling=false",
@@ -221,6 +225,10 @@ func TestOverrideViaCLIFlags(t *testing.T) {
 		APIServerHost:      "kube-apiserver.internal",
 		KubeConfigFilePath: "/path/to/kubeconfig",
 
+		DisableIngressNetworkingV1:      true,
+		DisableIngressNetworkingV1beta1: true,
+		DisableIngressExtensionsV1beta1: true,
+
 		LogLevel:  "info",
 		LogFormat: "json",
 
@@ -251,6 +259,10 @@ func TestOverrideViaEnvVars(t *testing.T) {
 		"CONTROLLER_LOG_LEVEL": "panic",
 
 		"CONTROLLER_KONG_CUSTOM_ENTITIES_SECRET": "foons/barsecretname",
+
+		"CONTROLLER_DISABLE_INGRESS_EXTENSIONSV1BETA1": "true",
+		"CONTROLLER_DISABLE_INGRESS_NETWORKINGV1BETA1": "true",
+		"CONTROLLER_DISABLE_INGRESS_NETWORKINGV1":      "true",
 	}
 	for k, v := range envs {
 		os.Setenv(k, v)
@@ -290,6 +302,10 @@ func TestOverrideViaEnvVars(t *testing.T) {
 		APIServerHost:      "",
 		KubeConfigFilePath: "",
 
+		DisableIngressNetworkingV1:      true,
+		DisableIngressNetworkingV1beta1: true,
+		DisableIngressExtensionsV1beta1: true,
+
 		LogLevel:  "panic",
 		LogFormat: "text",
 

cli/ingress-controller/flags.go

@@ -83,6 +83,11 @@ type cliConfig struct {
 	APIServerHost      string
 	KubeConfigFilePath string
 
+	// Allowed Ingress resource versions
+	DisableIngressExtensionsV1beta1 bool
+	DisableIngressNetworkingV1beta1 bool
+	DisableIngressNetworkingV1      bool
+
 	// Performance
 	EnableProfiling bool
 
@@ -226,6 +231,17 @@ Kubernetes cluster and local discovery is attempted.`)
 	flags.String("kubeconfig", "", "Path to kubeconfig file with "+
 		"authorization and master location information.")
 
+	// Allowed Ingress resource versions
+	flags.Bool("disable-ingress-extensionsv1beta1", false,
+		`If set, the ingress controller won't try extensions/v1beta1 when negotiating the newest supported
+Ingress API with Kubernetes.`)
+	flags.Bool("disable-ingress-networkingv1beta1", false,
+		`If set, the ingress controller won't try networking.k8s.io/v1beta1 when negotiating the newest supported
+Ingress API with Kubernetes.`)
+	flags.Bool("disable-ingress-networkingv1", false,
+		`If set, the ingress controller won't try networking/v1 when negotiating the newest supported
+Ingress API with Kubernetes.`)
+
 	// Misc
 	flags.Bool("profiling", true, `Enable profiling via web interface host:port/debug/pprof/`)
 	flags.Bool("version", false,
@@ -350,6 +366,11 @@ func parseFlags() (cliConfig, error) {
 	config.APIServerHost = viper.GetString("apiserver-host")
 	config.KubeConfigFilePath = viper.GetString("kubeconfig")
 
+	// Disabled Ingress resource versions
+	config.DisableIngressExtensionsV1beta1 = viper.GetBool("disable-ingress-extensionsv1beta1")
+	config.DisableIngressNetworkingV1beta1 = viper.GetBool("disable-ingress-networkingv1beta1")
+	config.DisableIngressNetworkingV1 = viper.GetBool("disable-ingress-networkingv1")
+
 	// Misc
 	config.EnableProfiling = viper.GetBool("profiling")
 	config.ShowVersion = viper.GetBool("version")

cli/ingress-controller/main.go

@@ -307,13 +307,6 @@ func main() {
 	}
 	controllerConfig.Kong.Client = kongClient
 
-	err = discovery.ServerSupportsVersion(kubeClient.Discovery(), schema.GroupVersion{
-		Group:   "networking.k8s.io",
-		Version: "v1beta1",
-	})
-	if err == nil {
-		controllerConfig.UseNetworkingV1beta1 = true
-	}
 	coreInformerFactory := informers.NewSharedInformerFactoryWithOptions(
 		kubeClient,
 		cliConfig.SyncPeriod,
@@ -351,18 +344,41 @@ func main() {
 		UpdateCh: updateChannel,
 	}
 
+	var preferredIngressAPIs []utils.IngressAPI
+	if !cliConfig.DisableIngressNetworkingV1 {
+		preferredIngressAPIs = append(preferredIngressAPIs, utils.NetworkingV1)
+	}
+	if !cliConfig.DisableIngressNetworkingV1beta1 {
+		preferredIngressAPIs = append(preferredIngressAPIs, utils.NetworkingV1beta1)
+	}
+	if !cliConfig.DisableIngressExtensionsV1beta1 {
+		preferredIngressAPIs = append(preferredIngressAPIs, utils.ExtensionsV1beta1)
+	}
+	controllerConfig.IngressAPI, err = utils.NegotiateResourceAPI(kubeClient, "Ingress", preferredIngressAPIs)
+	if err != nil {
+		log.Fatalf("NegotiateIngressAPI failed: %v, tried: %+v", err, preferredIngressAPIs)
+	}
+	log.Infof("chosen Ingress API version: %v", controllerConfig.IngressAPI)
+
 	var informers []cache.SharedIndexInformer
 	var cacheStores store.CacheStores
 
 	var ingInformer cache.SharedIndexInformer
-	if controllerConfig.UseNetworkingV1beta1 {
+	switch controllerConfig.IngressAPI {
+	case utils.NetworkingV1:
+		ingInformer = coreInformerFactory.Networking().V1().Ingresses().Informer()
+		cacheStores.IngressV1 = ingInformer.GetStore()
+		cacheStores.IngressV1beta1 = newEmptyStore()
+	case utils.NetworkingV1beta1:
 		ingInformer = coreInformerFactory.Networking().V1beta1().Ingresses().Informer()
-	} else {
+		cacheStores.IngressV1 = newEmptyStore()
+		cacheStores.IngressV1beta1 = ingInformer.GetStore()
+	case utils.ExtensionsV1beta1:
 		ingInformer = coreInformerFactory.Extensions().V1beta1().Ingresses().Informer()
+		cacheStores.IngressV1 = newEmptyStore()
+		cacheStores.IngressV1beta1 = ingInformer.GetStore()
 	}
-
 	ingInformer.AddEventHandler(reh)
-	cacheStores.Ingress = ingInformer.GetStore()
 	informers = append(informers, ingInformer)
 
 	endpointsInformer := coreInformerFactory.Core().V1().Endpoints().Informer()

cli/ingress-controller/util.go

@@ -2,12 +2,14 @@ package main
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"regexp"
 	"strings"
 
 	"github.com/blang/semver"
 	"github.com/kong/go-kong/kong"
+	"k8s.io/client-go/tools/cache"
 )
 
 func getSemVerVer(v string) (semver.Version, error) {
@@ -57,3 +59,7 @@ func createWorkspace(ctx context.Context, client *kong.Client, workspace string)
 	_, err = client.Do(ctx, req, nil)
 	return err
 }
+
+func newEmptyStore() cache.Store {
+	return cache.NewStore(func(interface{}) (string, error) { return "", errors.New("this store cannot add elements") })
+}

cli/ingress-controller/util_test.go

@@ -1,6 +1,8 @@
 package main
 
-import "testing"
+import (
+	"testing"
+)
 
 func TestFixVersion(t *testing.T) {
 	validVersions := map[string]string{

internal/ingress/controller/controller.go

@@ -31,6 +31,7 @@ import (
 	"github.com/kong/kubernetes-ingress-controller/internal/ingress/status"
 	"github.com/kong/kubernetes-ingress-controller/internal/ingress/store"
 	"github.com/kong/kubernetes-ingress-controller/internal/ingress/task"
+	"github.com/kong/kubernetes-ingress-controller/internal/ingress/utils"
 	configurationv1 "github.com/kong/kubernetes-ingress-controller/pkg/apis/configuration/v1"
 	configClientSet "github.com/kong/kubernetes-ingress-controller/pkg/client/configuration/clientset/versioned"
 	"github.com/mitchellh/mapstructure"
@@ -87,7 +88,8 @@ type Configuration struct {
 	UpdateStatusOnShutdown bool
 	ElectionID             string
 
-	UseNetworkingV1beta1        bool
+	IngressAPI utils.IngressAPI
+
 	EnableKnativeIngressSupport bool
 
 	Logger logrus.FieldLogger
@@ -180,7 +182,7 @@ func NewKongController(ctx context.Context,
 			PublishStatusAddress:   config.PublishStatusAddress,
 			IngressLister:          n.store,
 			UpdateStatusOnShutdown: config.UpdateStatusOnShutdown,
-			UseNetworkingV1beta1:   config.UseNetworkingV1beta1,
+			IngressAPI:             config.IngressAPI,
 			OnStartedLeading: func() {
 				// force a sync
 				n.syncQueue.Enqueue(&networking.Ingress{})

internal/ingress/controller/parser/kongstate/kongstate.go

@@ -147,24 +147,12 @@ func (ks *KongState) FillOverrides(log logrus.FieldLogger, s store.Storer) {
 		for j := 0; j < len(ks.Services[i].Routes); j++ {
 			var kongIngress *configurationv1.KongIngress
 			var err error
-			if ks.Services[i].Routes[j].IsTCP {
-				kongIngress, err = getKongIngressFromTCPIngress(s,
-					&ks.Services[i].Routes[j].TCPIngress)
-				if err != nil {
-					log.WithFields(logrus.Fields{
-						"tcpingress_name":      ks.Services[i].Routes[j].TCPIngress.Name,
-						"tcpingress_namespace": ks.Services[i].Routes[j].TCPIngress.Namespace,
-					}).Errorf("failed to fetch KongIngress resource for Ingress: %v", err)
-				}
-			} else {
-				kongIngress, err = getKongIngressFromIngress(s,
-					&ks.Services[i].Routes[j].Ingress)
-				if err != nil {
-					log.WithFields(logrus.Fields{
-						"ingress_name":      ks.Services[i].Routes[j].Ingress.Name,
-						"ingress_namespace": ks.Services[i].Routes[j].Ingress.Namespace,
-					}).Errorf("failed to fetch KongIngress resource for Ingress: %v", err)
-				}
+			kongIngress, err = getKongIngressFromObjectMeta(s, &ks.Services[i].Routes[j].Ingress)
+			if err != nil {
+				log.WithFields(logrus.Fields{
+					"resource_name":      ks.Services[i].Routes[j].Ingress.Name,
+					"resource_namespace": ks.Services[i].Routes[j].Ingress.Namespace,
+				}).Errorf("failed to fetch KongIngress resource: %v", err)
 			}
 
 			ks.Services[i].Routes[j].override(log, kongIngress)
@@ -230,7 +218,7 @@ func (ks *KongState) getPluginRelations() map[string]util.ForeignRelations {
 		// route
 		for j := range ks.Services[i].Routes {
 			ingress := ks.Services[i].Routes[j].Ingress
-			pluginList := annotations.ExtractKongPluginsFromAnnotations(ingress.GetAnnotations())
+			pluginList := annotations.ExtractKongPluginsFromAnnotations(ingress.Annotations)
 			for _, pluginName := range pluginList {
 				addRouteRelation(ingress.Namespace, pluginName, *ks.Services[i].Routes[j].Name)
 			}

internal/ingress/controller/parser/kongstate/kongstate_test.go

@@ -9,7 +9,6 @@ import (
 	"github.com/kong/kubernetes-ingress-controller/internal/ingress/controller/parser/util"
 	configurationv1 "github.com/kong/kubernetes-ingress-controller/pkg/apis/configuration/v1"
 	corev1 "k8s.io/api/core/v1"
-	networking "k8s.io/api/networking/v1beta1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
@@ -92,13 +91,11 @@ func Test_getPluginRelations(t *testing.T) {
 									Route: kong.Route{
 										Name: kong.String("foo-route"),
 									},
-									Ingress: networking.Ingress{
-										ObjectMeta: metav1.ObjectMeta{
-											Name:      "some-ingress",
-											Namespace: "ns2",
-											Annotations: map[string]string{
-												annotations.DeprecatedPluginsKey: "foo,bar",
-											},
+									Ingress: util.K8sObjectInfo{
+										Name:      "some-ingress",
+										Namespace: "ns2",
+										Annotations: map[string]string{
+											annotations.DeprecatedPluginsKey: "foo,bar",
 										},
 									},
 								},
@@ -126,27 +123,23 @@ func Test_getPluginRelations(t *testing.T) {
 									Route: kong.Route{
 										Name: kong.String("foo-route"),
 									},
-									Ingress: networking.Ingress{
-										ObjectMeta: metav1.ObjectMeta{
-											Name:      "some-ingress",
-											Namespace: "ns2",
-											Annotations: map[string]string{
-												annotations.DeprecatedPluginsKey: "foo,bar",
-											},
+									Ingress: util.K8sObjectInfo{
+										Name:      "some-ingress",
+										Namespace: "ns2",
+										Annotations: map[string]string{
+											annotations.DeprecatedPluginsKey: "foo,bar",
 										},
 									},
 								},
 								{
 									Route: kong.Route{
 										Name: kong.String("bar-route"),
 									},
-									Ingress: networking.Ingress{
-										ObjectMeta: metav1.ObjectMeta{
-											Name:      "some-ingress",
-											Namespace: "ns2",
-											Annotations: map[string]string{
-												annotations.DeprecatedPluginsKey: "bar,baz",
-											},
+									Ingress: util.K8sObjectInfo{
+										Name:      "some-ingress",
+										Namespace: "ns2",
+										Annotations: map[string]string{
+											annotations.DeprecatedPluginsKey: "bar,baz",
 										},
 									},
 								},
@@ -224,27 +217,23 @@ func Test_getPluginRelations(t *testing.T) {
 									Route: kong.Route{
 										Name: kong.String("foo-route"),
 									},
-									Ingress: networking.Ingress{
-										ObjectMeta: metav1.ObjectMeta{
-											Name:      "some-ingress",
-											Namespace: "ns2",
-											Annotations: map[string]string{
-												annotations.DeprecatedPluginsKey: "foo,bar",
-											},
+									Ingress: util.K8sObjectInfo{
+										Name:      "some-ingress",
+										Namespace: "ns2",
+										Annotations: map[string]string{
+											annotations.DeprecatedPluginsKey: "foo,bar",
 										},
 									},
 								},
 								{
 									Route: kong.Route{
 										Name: kong.String("bar-route"),
 									},
-									Ingress: networking.Ingress{
-										ObjectMeta: metav1.ObjectMeta{
-											Name:      "some-ingress",
-											Namespace: "ns2",
-											Annotations: map[string]string{
-												annotations.DeprecatedPluginsKey: "bar,baz",
-											},
+									Ingress: util.K8sObjectInfo{
+										Name:      "some-ingress",
+										Namespace: "ns2",
+										Annotations: map[string]string{
+											annotations.DeprecatedPluginsKey: "bar,baz",
 										},
 									},
 								},