diff --git a/changelog/v1.18.4/deleg-label.yaml b/changelog/v1.18.4/deleg-label.yaml new file mode 100644 index 00000000000..db130cc42db --- /dev/null +++ b/changelog/v1.18.4/deleg-label.yaml @@ -0,0 +1,17 @@ +changelog: + - type: FIX + issueLink: https://github.com/solo-io/solo-projects/issues/7626 + resolvesIssue: false + description: | + gateway2: allow route delegation using wellknown label + + There is a product requirement to enable users to use + a label to select HTTPRoutes to delegate to instead + of GVK ref to other HTTPRoutes (includes wildcards). + + To strike a balance between flexibility and performance, + this change implements the proposal to use a well known + label `delegation.gateway.solo.io/label=` to + allow users to delegate to other HTTPRoutes using a label. + HTTPRoutes are indexed using this well known label key that + enable O(1) lookups of routes matching this label value. diff --git a/projects/gateway2/controller/controller.go b/projects/gateway2/controller/controller.go index 98c4e410665..3f72dc7c2d3 100644 --- a/projects/gateway2/controller/controller.go +++ b/projects/gateway2/controller/controller.go @@ -123,6 +123,10 @@ func (c *controllerBuilder) addIndexes(ctx context.Context) error { if err := c.cfg.Mgr.GetFieldIndexer().IndexField(ctx, &apiv1.HTTPRoute{}, query.HttpRouteTargetField, query.IndexerByObjType); err != nil { errs = append(errs, err) } + // Index HTTPRoutes by the delegation.gateway.solo.io/label label value to lookup delegatee routes using the label + if err := c.cfg.Mgr.GetFieldIndexer().IndexField(ctx, &apiv1.HTTPRoute{}, query.HttpRouteDelegatedLabelSelector, query.IndexByHTTPRouteDelegationLabelSelector); err != nil { + errs = append(errs, err) + } // Index for ReferenceGrant if err := c.cfg.Mgr.GetFieldIndexer().IndexField(ctx, &apiv1beta1.ReferenceGrant{}, query.ReferenceGrantFromField, query.IndexerByObjType); err != nil { diff --git a/projects/gateway2/query/httproute.go b/projects/gateway2/query/httproute.go index 1fca5e20ea3..90959d8201b 100644 --- a/projects/gateway2/query/httproute.go +++ b/projects/gateway2/query/httproute.go @@ -257,8 +257,8 @@ func (r *gatewayQueries) getDelegatedChildren( for _, parentRule := range parent.Spec.Rules { var refChildren []*RouteInfo for _, backendRef := range parentRule.BackendRefs { - // Check if the backend reference is an HTTPRoute - if !backendref.RefIsHTTPRoute(backendRef.BackendObjectReference) { + // Check if the backend delegated route reference + if !backendref.RefIsDelegatedHTTPRoute(backendRef.BackendObjectReference) { continue } // Fetch child routes based on the backend reference @@ -302,35 +302,41 @@ func (r *gatewayQueries) fetchChildRoutes( backendRef gwv1.HTTPBackendRef, ) ([]gwv1.HTTPRoute, error) { delegatedNs := parentNamespace - if !backendref.RefIsHTTPRoute(backendRef.BackendObjectReference) { - return nil, nil - } // Use the namespace specified in the backend reference if available if backendRef.Namespace != nil { delegatedNs = string(*backendRef.Namespace) } var refChildren []gwv1.HTTPRoute - if string(backendRef.Name) == "" || string(backendRef.Name) == "*" { - // Handle wildcard references by listing all HTTPRoutes in the specified namespace - var hrlist gwv1.HTTPRouteList - err := r.client.List(ctx, &hrlist, client.InNamespace(delegatedNs)) - if err != nil { - return nil, err - } - refChildren = append(refChildren, hrlist.Items...) - } else { - // Lookup a specific child route by its name - delegatedRef := types.NamespacedName{ - Namespace: delegatedNs, - Name: string(backendRef.Name), + if backendref.RefIsHTTPRoute(backendRef.BackendObjectReference) { + if string(backendRef.Name) == "" || string(backendRef.Name) == "*" { + // Handle wildcard references by listing all HTTPRoutes in the specified namespace + var hrlist gwv1.HTTPRouteList + err := r.client.List(ctx, &hrlist, client.InNamespace(delegatedNs)) + if err != nil { + return nil, err + } + refChildren = hrlist.Items + } else { + // Lookup a specific child route by its name + delegatedRef := types.NamespacedName{ + Namespace: delegatedNs, + Name: string(backendRef.Name), + } + child := &gwv1.HTTPRoute{} + err := r.client.Get(ctx, delegatedRef, child) + if err != nil { + return nil, err + } + refChildren = append(refChildren, *child) } - child := &gwv1.HTTPRoute{} - err := r.client.Get(ctx, delegatedRef, child) + } else if backendref.RefIsHTTPRouteDelegationLabelSelector(backendRef.BackendObjectReference) { + var hrlist gwv1.HTTPRouteList + err := r.client.List(ctx, &hrlist, client.InNamespace(delegatedNs), client.MatchingFields{HttpRouteDelegatedLabelSelector: string(backendRef.Name)}) if err != nil { return nil, err } - refChildren = append(refChildren, *child) + refChildren = hrlist.Items } // Check if no child routes were resolved and log an error if needed if len(refChildren) == 0 { diff --git a/projects/gateway2/query/indexers.go b/projects/gateway2/query/indexers.go index 83d7cfb8fb5..ecf63d25c88 100644 --- a/projects/gateway2/query/indexers.go +++ b/projects/gateway2/query/indexers.go @@ -13,15 +13,17 @@ import ( ) const ( - HttpRouteTargetField = "http-route-target" - TcpRouteTargetField = "tcp-route-target" - ReferenceGrantFromField = "ref-grant-from" + HttpRouteTargetField = "http-route-target" + HttpRouteDelegatedLabelSelector = "http-route-delegated-label-selector" + TcpRouteTargetField = "tcp-route-target" + ReferenceGrantFromField = "ref-grant-from" ) // IterateIndices calls the provided function for each indexable object with the appropriate indexer function. func IterateIndices(f func(client.Object, string, client.IndexerFunc) error) error { return errors.Join( f(&gwv1.HTTPRoute{}, HttpRouteTargetField, IndexerByObjType), + f(&gwv1.HTTPRoute{}, HttpRouteDelegatedLabelSelector, IndexByHTTPRouteDelegationLabelSelector), f(&gwv1a2.TCPRoute{}, TcpRouteTargetField, IndexerByObjType), f(&gwv1b1.ReferenceGrant{}, ReferenceGrantFromField, IndexerByObjType), ) @@ -86,6 +88,15 @@ func IndexerByObjType(obj client.Object) []string { return results } +func IndexByHTTPRouteDelegationLabelSelector(obj client.Object) []string { + route := obj.(*gwv1.HTTPRoute) + value, ok := route.Labels[wellknown.RouteDelegationLabelSelector] + if !ok { + return nil + } + return []string{value} +} + // resolveNs resolves the namespace from an optional Namespace field. func resolveNs(ns *gwv1.Namespace) string { if ns == nil { diff --git a/projects/gateway2/translator/backendref/types.go b/projects/gateway2/translator/backendref/types.go index 560443d33d0..068e175f362 100644 --- a/projects/gateway2/translator/backendref/types.go +++ b/projects/gateway2/translator/backendref/types.go @@ -27,6 +27,18 @@ func RefIsHTTPRoute(ref gwv1.BackendObjectReference) bool { return (ref.Kind != nil && *ref.Kind == wellknown.HTTPRouteKind) && (ref.Group != nil && *ref.Group == gwv1.GroupName) } +// RefIsHTTPRouteDelegationLabelSelector checks if the BackendObjectReference is an HTTPRoute delegation label selector +// Parent routes may delegate to child routes using an HTTPRoute backend reference. +func RefIsHTTPRouteDelegationLabelSelector(ref gwv1.BackendObjectReference) bool { + return ref.Group != nil && ref.Kind != nil && (string(*ref.Group)+"/"+string(*ref.Kind)) == wellknown.RouteDelegationLabelSelector +} + +// RefIsDelegatedHTTPRoute checks if the BackendObjectReference is a delegated HTTPRoute +// selected by an HTTPRoute GVK reference or a delegation label selector. +func RefIsDelegatedHTTPRoute(ref gwv1.BackendObjectReference) bool { + return RefIsHTTPRoute(ref) || RefIsHTTPRouteDelegationLabelSelector(ref) +} + // ToString returns a string representation of the BackendObjectReference func ToString(ref gwv1.BackendObjectReference) string { var group, kind, namespace string diff --git a/projects/gateway2/translator/gateway_translator_test.go b/projects/gateway2/translator/gateway_translator_test.go index 0bb33a6cc57..9b881bf307e 100644 --- a/projects/gateway2/translator/gateway_translator_test.go +++ b/projects/gateway2/translator/gateway_translator_test.go @@ -329,4 +329,5 @@ var _ = DescribeTable("Route Delegation translator", Entry("RouteOptions prefer child override when allowed", "route_options_inheritance_child_override_allow.yaml"), Entry("RouteOptions multi level inheritance with child override when allowed", "route_options_multi_level_inheritance_override_allow.yaml"), Entry("RouteOptions multi level inheritance with partial child override", "route_options_multi_level_inheritance_override_partial.yaml"), + Entry("Label based delegation", "label_based.yaml"), ) diff --git a/projects/gateway2/translator/httproute/delegation_helpers.go b/projects/gateway2/translator/httproute/delegation_helpers.go index f76e4809f94..2c7544409c8 100644 --- a/projects/gateway2/translator/httproute/delegation_helpers.go +++ b/projects/gateway2/translator/httproute/delegation_helpers.go @@ -13,11 +13,6 @@ import ( gwv1 "sigs.k8s.io/gateway-api/apis/v1" ) -// inheritMatcherAnnotation is the annotation used on an child HTTPRoute that -// participates in a delegation chain to indicate that child route should inherit -// the route matcher from the parent route. -const inheritMatcherAnnotation = "delegation.gateway.solo.io/inherit-parent-matcher" - // filterDelegatedChildren filters the referenced children and their rules based // on parent matchers, filters their hostnames, and applies parent matcher // inheritance @@ -184,7 +179,7 @@ func isDelegatedRouteMatch( // shouldInheritMatcher returns true if the route indicates that it should inherit // its parent's matcher. func shouldInheritMatcher(route *gwv1.HTTPRoute) bool { - val, ok := route.Annotations[inheritMatcherAnnotation] + val, ok := route.Annotations[wellknown.InheritMatcherAnnotation] if !ok { return false } diff --git a/projects/gateway2/translator/httproute/gateway_http_route_translator.go b/projects/gateway2/translator/httproute/gateway_http_route_translator.go index cb38e2a4e68..8c4c03ad17f 100644 --- a/projects/gateway2/translator/httproute/gateway_http_route_translator.go +++ b/projects/gateway2/translator/httproute/gateway_http_route_translator.go @@ -329,7 +329,7 @@ func setRouteAction( for _, backendRef := range backendRefs { // If the backend is an HTTPRoute, it implies route delegation // for which delegated routes are recursively flattened and translated - if backendref.RefIsHTTPRoute(backendRef.BackendObjectReference) { + if backendref.RefIsDelegatedHTTPRoute(backendRef.BackendObjectReference) { delegates = true // Flatten delegated HTTPRoute references err := flattenDelegatedRoutes( diff --git a/projects/gateway2/translator/plugins/routeoptions/route_options_plugin.go b/projects/gateway2/translator/plugins/routeoptions/route_options_plugin.go index 03eea5fb2be..31a6c857c5c 100644 --- a/projects/gateway2/translator/plugins/routeoptions/route_options_plugin.go +++ b/projects/gateway2/translator/plugins/routeoptions/route_options_plugin.go @@ -28,16 +28,13 @@ import ( rtoptquery "github.com/solo-io/gloo/projects/gateway2/translator/plugins/routeoptions/query" "github.com/solo-io/gloo/projects/gateway2/translator/plugins/utils" "github.com/solo-io/gloo/projects/gateway2/translator/routeutils" + "github.com/solo-io/gloo/projects/gateway2/wellknown" "github.com/solo-io/gloo/projects/gloo/pkg/api/grpc/validation" gloov1 "github.com/solo-io/gloo/projects/gloo/pkg/api/v1" glooutils "github.com/solo-io/gloo/projects/gloo/pkg/utils" ) const ( - // policyOverrideAnnotation can be set by parent routes to allow child routes to override - // all (wildcard *) or specific fields (comma separated field names) in RouteOptions inherited from the parent route. - policyOverrideAnnotation = "delegation.gateway.solo.io/enable-policy-overrides" - // wildcardField is used to enable overriding all fields in RouteOptions inherited from the parent route. wildcardField = "*" ) @@ -131,7 +128,7 @@ func mergeOptionsForRoute( // and can only augment them during a merge such that fields unset in the higher // priority options can be merged in from the lower priority options. // In the case of delegated routes, a parent route can enable child routes to override - // all (wildcard *) or specific fields using the policyOverrideAnnotation. + // all (wildcard *) or specific fields using the wellknown.PolicyOverrideAnnotation. fieldsAllowedToOverride := sets.New[string]() // If the route already has options set, we should override/augment them. @@ -141,13 +138,13 @@ func mergeOptionsForRoute( // // By default, parent options (routeOptions) are preferred, unless the parent explicitly // enabled child routes (outputRoute.Options) to override parent options. - fieldsStr, delegatedPolicyOverride := route.Annotations[policyOverrideAnnotation] + fieldsStr, delegatedPolicyOverride := route.Annotations[wellknown.PolicyOverrideAnnotation] if delegatedPolicyOverride { delegatedFieldsToOverride := parseDelegationFieldOverrides(fieldsStr) if delegatedFieldsToOverride.Len() == 0 { // Invalid annotation value, so log an error but enforce the default behavior of preferring the parent options. contextutils.LoggerFrom(ctx).Errorf("invalid value %q for annotation %s on route %s; must be %s or a comma-separated list of field names", - fieldsStr, policyOverrideAnnotation, client.ObjectKeyFromObject(route), wildcardField) + fieldsStr, wellknown.PolicyOverrideAnnotation, client.ObjectKeyFromObject(route), wildcardField) } else { fieldsAllowedToOverride = delegatedFieldsToOverride } diff --git a/projects/gateway2/translator/plugins/routeoptions/route_options_plugin_test.go b/projects/gateway2/translator/plugins/routeoptions/route_options_plugin_test.go index 14269cb65e0..dfa9f4a770e 100644 --- a/projects/gateway2/translator/plugins/routeoptions/route_options_plugin_test.go +++ b/projects/gateway2/translator/plugins/routeoptions/route_options_plugin_test.go @@ -773,7 +773,7 @@ var _ = DescribeTable("mergeOptionsForRoute", Entry("override dst options with annotation: full override", &gwv1.HTTPRoute{ ObjectMeta: metav1.ObjectMeta{ - Annotations: map[string]string{policyOverrideAnnotation: "*"}, + Annotations: map[string]string{wellknown.PolicyOverrideAnnotation: "*"}, }, }, &v1.RouteOptions{ @@ -804,7 +804,7 @@ var _ = DescribeTable("mergeOptionsForRoute", Entry("override dst options with annotation: partial override", &gwv1.HTTPRoute{ ObjectMeta: metav1.ObjectMeta{ - Annotations: map[string]string{policyOverrideAnnotation: "*"}, + Annotations: map[string]string{wellknown.PolicyOverrideAnnotation: "*"}, }, }, &v1.RouteOptions{ @@ -837,7 +837,7 @@ var _ = DescribeTable("mergeOptionsForRoute", Entry("override dst options with annotation: no override", &gwv1.HTTPRoute{ ObjectMeta: metav1.ObjectMeta{ - Annotations: map[string]string{policyOverrideAnnotation: "*"}, + Annotations: map[string]string{wellknown.PolicyOverrideAnnotation: "*"}, }, }, &v1.RouteOptions{ @@ -860,7 +860,7 @@ var _ = DescribeTable("mergeOptionsForRoute", Entry("override dst options with annotation: specific fields", &gwv1.HTTPRoute{ ObjectMeta: metav1.ObjectMeta{ - Annotations: map[string]string{policyOverrideAnnotation: "faults,timeout"}, + Annotations: map[string]string{wellknown.PolicyOverrideAnnotation: "faults,timeout"}, }, }, &v1.RouteOptions{ @@ -895,7 +895,7 @@ var _ = DescribeTable("mergeOptionsForRoute", Entry("override and augment dst options with annotation: specific fields", &gwv1.HTTPRoute{ ObjectMeta: metav1.ObjectMeta{ - Annotations: map[string]string{policyOverrideAnnotation: "faults,timeout"}, + Annotations: map[string]string{wellknown.PolicyOverrideAnnotation: "faults,timeout"}, }, }, &v1.RouteOptions{ diff --git a/projects/gateway2/translator/testutils/inputs/delegation/label_based.yaml b/projects/gateway2/translator/testutils/inputs/delegation/label_based.yaml new file mode 100644 index 00000000000..e0949fc03cf --- /dev/null +++ b/projects/gateway2/translator/testutils/inputs/delegation/label_based.yaml @@ -0,0 +1,179 @@ +apiVersion: gateway.networking.k8s.io/v1 +kind: Gateway +metadata: + name: example-gateway + namespace: infra +spec: + gatewayClassName: example-gateway-class + listeners: + - name: http + protocol: HTTP + port: 80 +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: example-route + namespace: infra +spec: + parentRefs: + - name: example-gateway + hostnames: + - "example.com" + rules: + - backendRefs: + - name: example-svc + port: 80 + - matches: + - path: + type: PathPrefix + value: /a + backendRefs: + - group: delegation.gateway.solo.io + kind: label + name: a-label + namespace: a + - matches: + - path: + type: PathPrefix + value: /b + backendRefs: + - group: delegation.gateway.solo.io + kind: label + name: b-label + # namespace defaults to parent's namespace + - matches: + - path: + type: PathPrefix + value: /c + backendRefs: + - group: gateway.networking.k8s.io + kind: HTTPRoute + name: "*" + namespace: c +--- +apiVersion: v1 +kind: Service +metadata: + name: example-svc + namespace: infra +spec: + selector: + test: test + ports: + - protocol: TCP + port: 80 +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: route-a1 + namespace: a + labels: + delegation.gateway.solo.io/label: a-label +spec: + rules: + - matches: + - path: + type: Exact + value: /a/1 + backendRefs: + - name: svc-a + port: 8080 +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: route-a2 + namespace: a + labels: + delegation.gateway.solo.io/label: a-label +spec: + rules: + - matches: + - path: + type: Exact + value: /a/2 + backendRefs: + - name: svc-a + port: 8080 +--- +# route-a3 does not match the selected label so it should be ignored +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: route-a3 + namespace: a + labels: + delegation.gateway.solo.io/label: not-a-label +spec: + rules: + - matches: + - path: + type: Exact + value: /a/3 + backendRefs: + - name: svc-a + port: 8080 +--- +apiVersion: v1 +kind: Service +metadata: + name: svc-a + namespace: a +spec: + ports: + - protocol: TCP + port: 8080 +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: route-b + namespace: infra + labels: + delegation.gateway.solo.io/label: b-label +spec: + rules: + - matches: + - path: + type: RegularExpression + value: /b/.* + backendRefs: + - name: svc-b + port: 8080 +--- +apiVersion: v1 +kind: Service +metadata: + name: svc-b + namespace: infra +spec: + ports: + - protocol: TCP + port: 8080 +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: route-c + namespace: c +spec: + rules: + - matches: + - path: + type: RegularExpression + value: /c/.* + backendRefs: + - name: svc-c + port: 8080 +--- +apiVersion: v1 +kind: Service +metadata: + name: svc-c + namespace: c +spec: + ports: + - protocol: TCP + port: 8080 \ No newline at end of file diff --git a/projects/gateway2/translator/testutils/outputs/delegation/label_based.yaml b/projects/gateway2/translator/testutils/outputs/delegation/label_based.yaml new file mode 100644 index 00000000000..c3948303547 --- /dev/null +++ b/projects/gateway2/translator/testutils/outputs/delegation/label_based.yaml @@ -0,0 +1,85 @@ +--- +listeners: +- aggregateListener: + httpFilterChains: + - matcher: {} + virtualHostRefs: + - http~example_com + httpResources: + virtualHosts: + http~example_com: + domains: + - example.com + name: http~example_com + routes: + - matchers: + - exact: /a/1 + options: {} + name: httproute-route-a1-a-0-0 + routeAction: + single: + kube: + port: 8080 + ref: + name: svc-a + namespace: a + - matchers: + - exact: /a/2 + options: {} + name: httproute-route-a2-a-0-0 + routeAction: + single: + kube: + port: 8080 + ref: + name: svc-a + namespace: a + - matchers: + - regex: /b/.* + options: {} + name: httproute-route-b-infra-0-0 + routeAction: + single: + kube: + port: 8080 + ref: + name: svc-b + namespace: infra + - matchers: + - regex: /c/.* + options: {} + name: httproute-route-c-c-0-0 + routeAction: + single: + kube: + port: 8080 + ref: + name: svc-c + namespace: c + - matchers: + - prefix: / + options: {} + name: httproute-example-route-infra-0-0 + routeAction: + single: + kube: + port: 80 + ref: + name: example-svc + namespace: infra + + bindAddress: '::' + bindPort: 8080 + metadataStatic: + sources: + - resourceKind: gateway.networking.k8s.io/Gateway + resourceRef: + name: http + namespace: infra + name: http +metadata: + labels: + created_by: gloo-kube-gateway-api + gateway_namespace: infra + name: infra-example-gateway + namespace: gloo-system diff --git a/projects/gateway2/wellknown/delegation.go b/projects/gateway2/wellknown/delegation.go new file mode 100644 index 00000000000..d763f372c22 --- /dev/null +++ b/projects/gateway2/wellknown/delegation.go @@ -0,0 +1,15 @@ +package wellknown + +const ( + // RouteDelegationLabelSelector is the label used to select delegated HTTPRoutes + RouteDelegationLabelSelector = "delegation.gateway.solo.io/label" + + // InheritMatcherAnnotation is the annotation used on an child HTTPRoute that + // participates in a delegation chain to indicate that child route should inherit + // the route matcher from the parent route. + InheritMatcherAnnotation = "delegation.gateway.solo.io/inherit-parent-matcher" + + // PolicyOverrideAnnotation can be set by parent routes to allow child routes to override + // all (wildcard *) or specific fields (comma separated field names) in RouteOptions inherited from the parent route. + PolicyOverrideAnnotation = "delegation.gateway.solo.io/enable-policy-overrides" +) diff --git a/test/kubernetes/e2e/features/route_delegation/testdata/basic.yaml b/test/kubernetes/e2e/features/route_delegation/testdata/basic.yaml index 4e9c35feaad..0279ed0f79b 100644 --- a/test/kubernetes/e2e/features/route_delegation/testdata/basic.yaml +++ b/test/kubernetes/e2e/features/route_delegation/testdata/basic.yaml @@ -35,9 +35,9 @@ spec: type: PathPrefix value: /anything/team2 backendRefs: - - group: gateway.networking.k8s.io - kind: HTTPRoute - name: "*" + - group: delegation.gateway.solo.io + kind: label + name: team2 namespace: team2 --- apiVersion: gateway.networking.k8s.io/v1 @@ -60,6 +60,8 @@ kind: HTTPRoute metadata: name: svc2 namespace: team2 + labels: + delegation.gateway.solo.io/label: team2 spec: parentRefs: - name: root