Static NodePorts for Gateway API service ports (#10538)

Co-authored-by: Nathan Fudenberg <[email protected]>

changelog/v1.19.0-beta3/7504-gwapi-static-node-ports.yaml

@@ -0,0 +1,9 @@
+
+changelog:
+- type: NEW_FEATURE
+  issueLink: https://github.com/solo-io/solo-projects/issues/7504
+  resolvesIssue: true
+  description: > 
+    Add a new Ports field to the `GatewayParameters` `Kube.Service` Spec in order to allow admin users
+    to configure additional information about the ports that the Gateway should listen on. This is useful
+    if the user wants to specify a static `NodePort`

docs/content/reference/api/github.com/solo-io/gloo/projects/gateway2/api/v1alpha1/gateway_parameters.md

@@ -7756,6 +7756,13 @@ Resource Types:
           <br/>
         </td>
         <td>false</td>
+      </tr><tr>
+        <td><b><a href="#gatewayparametersspeckubeserviceportsindex">ports</a></b></td>
+        <td>[]object</td>
+        <td>
+          <br/>
+        </td>
+        <td>false</td>
       </tr><tr>
         <td><b>type</b></td>
         <td>string</td>
@@ -7767,6 +7774,40 @@ Resource Types:
 </table>
 
 
+### GatewayParameters.spec.kube.service.ports[index]
+<sup><sup>[↩ Parent](#gatewayparametersspeckubeservice)</sup></sup>
+
+
+
+
+
+<table>
+    <thead>
+        <tr>
+            <th>Name</th>
+            <th>Type</th>
+            <th>Description</th>
+            <th>Required</th>
+        </tr>
+    </thead>
+    <tbody><tr>
+        <td><b>port</b></td>
+        <td>integer</td>
+        <td>
+          <br/>
+        </td>
+        <td>true</td>
+      </tr><tr>
+        <td><b>nodePort</b></td>
+        <td>integer</td>
+        <td>
+          <br/>
+        </td>
+        <td>false</td>
+      </tr></tbody>
+</table>
+
+
 ### GatewayParameters.spec.kube.serviceAccount
 <sup><sup>[↩ Parent](#gatewayparametersspeckube)</sup></sup>
 

install/helm/gloo/crds/gateway.gloo.solo.io_gatewayparameters.yaml

@@ -2103,6 +2103,18 @@ spec:
                         additionalProperties:
                           type: string
                         type: object
+                      ports:
+                        items:
+                          properties:
+                            nodePort:
+                              type: integer
+                            port:
+                              type: integer
+                          required:
+                          - port
+                          type: object
+                        maxItems: 8
+                        type: array
                       type:
                         type: string
                     type: object

install/helm/gloo/templates/43-gatewayparameters.yaml

@@ -42,6 +42,10 @@ spec:
       extraAnnotations:
         {{- toYaml . | nindent 8 }}
       {{- end }}
+      {{- with ($gg.service).ports }}
+      ports:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
 {{- if $gg.serviceAccount }}
     serviceAccount:
       {{- with $gg.serviceAccount.extraLabels }}

projects/gateway2/api/v1alpha1/gateway_parameters_types.go

@@ -45,6 +45,20 @@ type GatewayParametersSpec struct {
 	SelfManaged *SelfManagedGateway `json:"selfManaged,omitempty"`
 }
 
+func (in *GatewayParametersSpec) GetKube() *KubernetesProxyConfig {
+	if in == nil {
+		return nil
+	}
+	return in.Kube
+}
+
+func (in *GatewayParametersSpec) GetSelfManaged() *SelfManagedGateway {
+	if in == nil {
+		return nil
+	}
+	return in.SelfManaged
+}
+
 // The current conditions of the GatewayParameters. This is not currently implemented.
 type GatewayParametersStatus struct {
 }

projects/gateway2/api/v1alpha1/kube_types.go

@@ -97,6 +97,12 @@ type Service struct {
 	// +kubebuilder:validation:Optional
 	ExtraAnnotations map[string]string `json:"extraAnnotations,omitempty"`
 
+	// Additional configuration for the service ports.
+	// The actual port numbers are specified in the Gateway resource.
+	//
+	// +optional
+	// +kubebuilder:validation:MaxItems=8
+	Ports []*Port `json:"ports"`
 	// External Traffic Policy on the Service object.
 	//
 	// +kubebuilder:validation:Optional
@@ -131,13 +137,47 @@ func (in *Service) GetExtraAnnotations() map[string]string {
 	return in.ExtraAnnotations
 }
 
+func (in *Service) GetPorts() []*Port {
+	if in == nil {
+		return nil
+	}
+	return in.Ports
+}
+
 func (in *Service) GetExternalTrafficPolicy() *corev1.ServiceExternalTrafficPolicy {
 	if in == nil {
 		return nil
 	}
 	return in.ExternalTrafficPolicy
 }
 
+type Port struct {
+	// The port number to match on the Gateway
+	//
+	// +kubebuilder:validation:Required
+	Port uint16 `json:"port"`
+
+	// The NodePort to be used for the service. If not specified, a random port
+	// will be assigned by the Kubernetes API server.
+	//
+	// +kubebuilder:validation:Optional
+	NodePort *uint16 `json:"nodePort,omitempty"`
+}
+
+func (in *Port) GetPort() uint16 {
+	if in == nil {
+		return 0
+	}
+	return in.Port
+}
+
+func (in *Port) GetNodePort() *uint16 {
+	if in == nil {
+		return nil
+	}
+	return in.NodePort
+}
+
 type ServiceAccount struct {
 	// Additional labels to add to the ServiceAccount object metadata.
 	//

projects/gateway2/deployer/deployer.go

@@ -262,7 +262,7 @@ func (d *Deployer) getValues(gw *api.Gateway, gwParam *v1alpha1.GatewayParameter
 			Name:             &gw.Name,
 			GatewayName:      &gw.Name,
 			GatewayNamespace: &gw.Namespace,
-			Ports:            getPortsValues(gw),
+			Ports:            getPortsValues(gw, gwParam),
 			Xds: &helmXds{
 				// The xds host/port MUST map to the Service definition for the Control Plane
 				// This is the socket address that the Proxy will connect to on startup, to receive xds updates

projects/gateway2/deployer/deployer_test.go

@@ -1255,9 +1255,46 @@ var _ = Describe("Deployer", func() {
 					port := svc.Spec.Ports[0]
 					Expect(port.Port).To(Equal(int32(80)))
 					Expect(port.TargetPort.IntVal).To(Equal(int32(8080)))
+					Expect(port.NodePort).To(Equal(int32(0)))
 					return nil
 				},
 			}),
+			Entry("static NodePort", &input{
+				dInputs:    defaultDeployerInputs(),
+				gw:         defaultGatewayWithGatewayParams(gwpOverrideName),
+				defaultGwp: defaultGatewayParams(),
+				overrideGwp: &gw2_v1alpha1.GatewayParameters{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      gwpOverrideName,
+						Namespace: defaultNamespace,
+					},
+					Spec: gw2_v1alpha1.GatewayParametersSpec{
+						Kube: &gw2_v1alpha1.KubernetesProxyConfig{
+							Service: &gw2_v1alpha1.Service{
+								Type: ptr.To(corev1.ServiceTypeNodePort),
+								Ports: []*gw2_v1alpha1.Port{
+									{
+										Port:     80,
+										NodePort: ptr.To(uint16(30000)),
+									},
+								},
+							},
+						},
+					},
+				},
+			}, &expectedOutput{
+				validationFunc: func(objs clientObjects, inp *input) error {
+					svc := objs.findService(defaultNamespace, defaultServiceName)
+					Expect(svc).NotTo(BeNil())
+
+					port := svc.Spec.Ports[0]
+					Expect(port.Port).To(Equal(int32(80)))
+					Expect(port.TargetPort.IntVal).To(Equal(int32(8080)))
+					Expect(port.NodePort).To(Equal(int32(30000)))
+					return nil
+				},
+			}),
+
 			Entry("duplicate ports", &input{
 				dInputs: defaultDeployerInputs(),
 				gw: &api.Gateway{

projects/gateway2/deployer/merge.go

@@ -442,6 +442,7 @@ func deepMergeService(dst, src *v1alpha1.Service) *v1alpha1.Service {
 
 	dst.ExtraLabels = deepMergeMaps(dst.GetExtraLabels(), src.GetExtraLabels())
 	dst.ExtraAnnotations = deepMergeMaps(dst.GetExtraAnnotations(), src.GetExtraAnnotations())
+	dst.Ports = deepMergeSlices(dst.GetPorts(), src.GetPorts())
 
 	if src.GetExternalTrafficPolicy() != nil {
 		dst.ExternalTrafficPolicy = src.GetExternalTrafficPolicy()

projects/gateway2/deployer/values.go

@@ -74,6 +74,7 @@ type helmPort struct {
 	Protocol   *string `json:"protocol,omitempty"`
 	Name       *string `json:"name,omitempty"`
 	TargetPort *uint16 `json:"targetPort,omitempty"`
+	NodePort   *uint16 `json:"nodePort,omitempty"`
 }
 
 type helmImage struct {

projects/gateway2/deployer/values_helpers.go

@@ -10,6 +10,7 @@ import (
 	"github.com/solo-io/gloo/projects/gateway2/api/v1alpha1"
 	"github.com/solo-io/gloo/projects/gateway2/ports"
 	"golang.org/x/exp/slices"
+	corev1 "k8s.io/api/core/v1"
 	"k8s.io/utils/ptr"
 	api "sigs.k8s.io/gateway-api/apis/v1"
 )
@@ -24,7 +25,7 @@ var ComponentLogLevelEmptyError = func(key string, value string) error {
 // Extract the listener ports from a Gateway. These will be used to populate:
 // 1. the ports exposed on the envoy container
 // 2. the ports exposed on the proxy service
-func getPortsValues(gw *api.Gateway) []helmPort {
+func getPortsValues(gw *api.Gateway, gwp *v1alpha1.GatewayParameters) []helmPort {
 	gwPorts := []helmPort{}
 	for _, l := range gw.Spec.Listeners {
 		listenerPort := uint16(l.Port)
@@ -38,11 +39,23 @@ func getPortsValues(gw *api.Gateway) []helmPort {
 		portName := string(l.Name)
 		protocol := "TCP"
 
+		// Search for static NodePort set from the GatewayParameters spec
+		// If not found the default value of `nil` will not render anything.
+		var nodePort *uint16 = nil
+		if gwp.Spec.GetKube().GetService().GetType() != nil && *(gwp.Spec.GetKube().GetService().GetType()) == corev1.ServiceTypeNodePort {
+			if idx := slices.IndexFunc(gwp.Spec.GetKube().GetService().GetPorts(), func(p *v1alpha1.Port) bool {
+				return p.GetPort() == uint16(listenerPort)
+			}); idx != -1 {
+				nodePort = ptr.To(uint16(*gwp.Spec.GetKube().GetService().GetPorts()[idx].GetNodePort()))
+			}
+		}
+
 		gwPorts = append(gwPorts, helmPort{
 			Port:       &listenerPort,
 			TargetPort: &targetPort,
 			Name:       &portName,
 			Protocol:   &protocol,
+			NodePort:   nodePort,
 		})
 	}
 	return gwPorts

projects/gateway2/helm/gloo-gateway/templates/gateway/proxy-deployment.yaml

@@ -392,6 +392,7 @@ spec:
     protocol: {{ $p.protocol }}
     targetPort: {{ $p.targetPort }}
     port: {{ $p.port }}
+    nodePort: {{ $p.nodePort }}
   {{- end }}
   selector:
     {{- include "gloo-gateway.gateway.selectorLabels" . | nindent 4 }}