Remove dependency on kube-rbac-proxy by adopting controller-runtime…

…'s native authn/authz for metrics

cmd/ironcore-controller-manager/main.go

@@ -4,6 +4,7 @@
 package main
 
 import (
+	"crypto/tls"
 	"flag"
 	"fmt"
 	"os"
@@ -28,6 +29,7 @@ import (
 	quotaevaluatorironcore "github.com/ironcore-dev/ironcore/internal/quota/evaluator/ironcore"
 	"github.com/ironcore-dev/ironcore/utils/quota"
 	"k8s.io/utils/lru"
+	"sigs.k8s.io/controller-runtime/pkg/metrics/filters"
 	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
 
 	// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
@@ -100,13 +102,21 @@ func init() {
 
 func main() {
 	var metricsAddr string
+	var secureMetrics bool
+	var enableHTTP2 bool
 	var enableLeaderElection bool
 	var probeAddr string
 	var prefixAllocationTimeout time.Duration
 	var volumeBindTimeout time.Duration
 	var virtualIPBindTimeout time.Duration
 	var networkInterfaceBindTimeout time.Duration
-	flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.")
+	var tlsOpts []func(*tls.Config)
+	flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "The address the metrics endpoint binds to. "+
+		"Use :8443 for HTTPS or :8080 for HTTP, or leave as 0 to disable the metrics service.")
+	flag.BoolVar(&secureMetrics, "metrics-secure", true,
+		"If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.")
+	flag.BoolVar(&enableHTTP2, "enable-http2", false,
+		"If set, HTTP/2 will be enabled for the metrics and webhook servers")
 	flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
 	flag.BoolVar(&enableLeaderElection, "leader-elect", false,
 		"Enable leader election for controller manager. "+
@@ -166,10 +176,47 @@ func main() {
 	ctrl.SetLogger(logger)
 	ctx := ctrl.SetupSignalHandler()
 
+	// if the enable-http2 flag is false (the default), http/2 should be disabled
+	// due to its vulnerabilities. More specifically, disabling http/2 will
+	// prevent from being vulnerable to the HTTP/2 Stream Cancellation and
+	// Rapid Reset CVEs. For more information see:
+	// - https://github.com/advisories/GHSA-qppj-fm5r-hxr3
+	// - https://github.com/advisories/GHSA-4374-p667-p6c8
+	disableHTTP2 := func(c *tls.Config) {
+		setupLog.Info("disabling http/2")
+		c.NextProtos = []string{"http/1.1"}
+	}
+
+	if !enableHTTP2 {
+		tlsOpts = append(tlsOpts, disableHTTP2)
+	}
+
+	// Metrics endpoint is enabled in 'config/default/kustomization.yaml'. The Metrics options configure the server.
+	// More info:
+	// - https://pkg.go.dev/sigs.k8s.io/[email protected]/pkg/metrics/server
+	// - https://book.kubebuilder.io/reference/metrics.html
+	metricsServerOptions := metricsserver.Options{
+		BindAddress:   metricsAddr,
+		SecureServing: secureMetrics,
+		TLSOpts:       tlsOpts,
+	}
+
+	if secureMetrics {
+		// FilterProvider is used to protect the metrics endpoint with authn/authz.
+		// These configurations ensure that only authorized users and service accounts
+		// can access the metrics endpoint. The RBAC are configured in 'config/controller/rbac/kustomization.yaml'. More info:
+		// https://pkg.go.dev/sigs.k8s.io/[email protected]/pkg/metrics/filters#WithAuthenticationAndAuthorization
+		metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization
+
+		// TODO(user): If CertDir, CertName, and KeyName are not specified, controller-runtime will automatically
+		// generate self-signed certificates for the metrics server. While convenient for development and testing,
+		// this setup is not recommended for production.
+	}
+
 	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
 		Logger:                 logger,
 		Scheme:                 scheme,
-		Metrics:                metricsserver.Options{BindAddress: metricsAddr},
+		Metrics:                metricsServerOptions,
 		HealthProbeBindAddress: probeAddr,
 		LeaderElection:         enableLeaderElection,
 		LeaderElectionID:       "d0ae00be.ironcore.dev",

config/bucketpoollet-broker/broker-rbac/kustomization.yaml

@@ -10,10 +10,13 @@ resources: # All RBAC will be applied under this service account in
   - cluster_role_binding.yaml
   - leader_election_role.yaml
   - leader_election_role_binding.yaml
-  # Comment the following 4 lines if you want to disable
-  # the auth proxy (https://github.com/brancz/kube-rbac-proxy)
-  # which protects your /metrics endpoint.
-  - auth_proxy_service.yaml
-  - auth_proxy_role.yaml
-  - auth_proxy_role_binding.yaml
-  - auth_proxy_client_clusterrole.yaml
+  # The following RBAC configurations are used to protect
+  # the metrics endpoint with authn/authz. These configurations
+  # ensure that only authorized users and service accounts
+  # can access the metrics endpoint. Comment the following
+  # permissions if you want to disable this protection.
+  # More info: https://book.kubebuilder.io/reference/metrics.html
+  - metrics_service.yaml
+  - metrics_auth_role.yaml
+  - metrics_auth_role_binding.yaml
+  - metrics_reader_role.yaml

config/bucketpoollet-broker/broker-rbac/auth_proxy_role.yaml → config/bucketpoollet-broker/broker-rbac/metrics_auth_role.yaml

@@ -1,7 +1,7 @@
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
-  name: proxy-role
+  name: metrics-auth-role
 rules:
 - apiGroups:
   - authentication.k8s.io

config/machinepoollet-broker/broker-rbac/auth_proxy_role_binding.yaml → config/bucketpoollet-broker/broker-rbac/metrics_auth_role_binding.yaml

@@ -1,11 +1,11 @@
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
-  name: proxy-rolebinding
+  name: metrics-auth-rolebinding
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: ClusterRole
-  name: proxy-role
+  name: metrics-auth-role
 subjects:
 - kind: ServiceAccount
   name: controller-manager

config/machinepoollet-broker/broker-rbac/auth_proxy_service.yaml → config/bucketpoollet-broker/broker-rbac/metrics_service.yaml

@@ -9,6 +9,7 @@ spec:
   ports:
   - name: https
     port: 8443
-    targetPort: https
+    protocol: TCP 
+    targetPort: 8443
   selector:
     control-plane: controller-manager

config/bucketpoollet-broker/default/kustomization.yaml

@@ -21,7 +21,7 @@ resources:
 #- ../prometheus
 
 patchesStrategicMerge:
-  - manager_auth_proxy_patch.yaml
+  - manager_metrics_patch.yaml
 
   # Mount the controller config file for loading manager configurations
   # through a ComponentConfig type

config/volumepoollet-broker/default/manager_auth_proxy_patch.yaml → config/bucketpoollet-broker/default/manager_metrics_patch.yaml

@@ -9,18 +9,8 @@ spec:
   template:
     spec:
       containers:
-        - name: kube-rbac-proxy
-          image: gcr.io/kubebuilder/kube-rbac-proxy:v0.8.0
-          args:
-            - "--secure-listen-address=0.0.0.0:8443"
-            - "--upstream=http://127.0.0.1:8080/"
-            - "--logtostderr=true"
-            - "--v=10"
-          ports:
-            - containerPort: 8443
-              name: https
         - name: manager
           args:
             - "--health-probe-bind-address=:8081"
-            - "--metrics-bind-address=127.0.0.1:8080"
+            - "--metrics-bind-address=:8443"
             - "--leader-elect"

config/controller/default/kustomization.yaml

@@ -21,7 +21,7 @@ resources:
 #- ../prometheus
 
 patchesStrategicMerge:
-  - manager_auth_proxy_patch.yaml
+  - manager_metrics_patch.yaml
 
   # Mount the controller config file for loading manager configurations
   # through a ComponentConfig type

config/controller/default/manager_auth_proxy_patch.yaml → config/controller/default/manager_metrics_patch.yaml

@@ -9,19 +9,9 @@ spec:
   template:
     spec:
       containers:
-        - name: kube-rbac-proxy
-          image: gcr.io/kubebuilder/kube-rbac-proxy:v0.8.0
-          args:
-            - "--secure-listen-address=0.0.0.0:8443"
-            - "--upstream=http://127.0.0.1:8080/"
-            - "--logtostderr=true"
-            - "--v=10"
-          ports:
-            - containerPort: 8443
-              name: https
         - name: manager
           args:
             - "--health-probe-bind-address=:8081"
-            - "--metrics-bind-address=127.0.0.1:8080"
+            - "--metrics-bind-address=:8443"
             - "--leader-elect"
             - "--controllers=*"

config/controller/rbac/kustomization.yaml

@@ -8,10 +8,13 @@ resources: # All RBAC will be applied under this service account in
   - role_binding.yaml
   - leader_election_role.yaml
   - leader_election_role_binding.yaml
-  # Comment the following 4 lines if you want to disable
-  # the auth proxy (https://github.com/brancz/kube-rbac-proxy)
-  # which protects your /metrics endpoint.
-  - auth_proxy_service.yaml
-  - auth_proxy_role.yaml
-  - auth_proxy_role_binding.yaml
-  - auth_proxy_client_clusterrole.yaml
+  # The following RBAC configurations are used to protect
+  # the metrics endpoint with authn/authz. These configurations
+  # ensure that only authorized users and service accounts
+  # can access the metrics endpoint. Comment the following
+  # permissions if you want to disable this protection.
+  # More info: https://book.kubebuilder.io/reference/metrics.html
+  - metrics_service.yaml
+  - metrics_auth_role.yaml
+  - metrics_auth_role_binding.yaml
+  - metrics_reader_role.yaml

config/machinepoollet-broker/broker-rbac/auth_proxy_role.yaml → config/controller/rbac/metrics_auth_role.yaml

@@ -1,7 +1,7 @@
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
-  name: proxy-role
+  name: metrics-auth-role
 rules:
 - apiGroups:
   - authentication.k8s.io

config/bucketpoollet-broker/broker-rbac/auth_proxy_role_binding.yaml → config/controller/rbac/metrics_auth_role_binding.yaml

@@ -1,11 +1,11 @@
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
-  name: proxy-rolebinding
+  name: metrics-auth-rolebinding
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: ClusterRole
-  name: proxy-role
+  name: metrics-auth-role
 subjects:
 - kind: ServiceAccount
   name: controller-manager

config/volumepoollet-broker/broker-rbac/auth_proxy_service.yaml → config/controller/rbac/metrics_service.yaml

@@ -9,6 +9,7 @@ spec:
   ports:
   - name: https
     port: 8443
-    targetPort: https
+    protocol: TCP 
+    targetPort: 8443
   selector:
     control-plane: controller-manager

config/machinepoollet-broker/broker-rbac/kustomization.yaml

@@ -10,10 +10,13 @@ resources: # All RBAC will be applied under this service account in
   - cluster_role_binding.yaml
   - leader_election_role.yaml
   - leader_election_role_binding.yaml
-  # Comment the following 4 lines if you want to disable
-  # the auth proxy (https://github.com/brancz/kube-rbac-proxy)
-  # which protects your /metrics endpoint.
-  - auth_proxy_service.yaml
-  - auth_proxy_role.yaml
-  - auth_proxy_role_binding.yaml
-  - auth_proxy_client_clusterrole.yaml
+  # The following RBAC configurations are used to protect
+  # the metrics endpoint with authn/authz. These configurations
+  # ensure that only authorized users and service accounts
+  # can access the metrics endpoint. Comment the following
+  # permissions if you want to disable this protection.
+  # More info: https://book.kubebuilder.io/reference/metrics.html
+  - metrics_service.yaml
+  - metrics_auth_role.yaml
+  - metrics_auth_role_binding.yaml
+  - metrics_reader_role.yaml

config/volumepoollet-broker/broker-rbac/auth_proxy_role.yaml → config/machinepoollet-broker/broker-rbac/metrics_auth_role.yaml

@@ -1,7 +1,7 @@
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
-  name: proxy-role
+  name: metrics-auth-role
 rules:
 - apiGroups:
   - authentication.k8s.io

config/controller/rbac/auth_proxy_role_binding.yaml → config/machinepoollet-broker/broker-rbac/metrics_auth_role_binding.yaml

@@ -1,11 +1,11 @@
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
-  name: proxy-rolebinding
+  name: metrics-auth-rolebinding
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: ClusterRole
-  name: proxy-role
+  name: metrics-auth-role
 subjects:
 - kind: ServiceAccount
   name: controller-manager

config/bucketpoollet-broker/broker-rbac/auth_proxy_service.yaml → config/machinepoollet-broker/broker-rbac/metrics_service.yaml

@@ -9,6 +9,7 @@ spec:
   ports:
   - name: https
     port: 8443
-    targetPort: https
+    protocol: TCP 
+    targetPort: 8443
   selector:
     control-plane: controller-manager

config/machinepoollet-broker/default/kustomization.yaml

@@ -21,7 +21,7 @@ resources:
 #- ../prometheus
 
 patchesStrategicMerge:
-- manager_auth_proxy_patch.yaml
+- manager_metrics_patch.yaml
 
 # Mount the controller config file for loading manager configurations
 # through a ComponentConfig type

config/machinepoollet-broker/default/manager_auth_proxy_patch.yaml → config/machinepoollet-broker/default/manager_metrics_patch.yaml

@@ -9,20 +9,10 @@ spec:
   template:
     spec:
       containers:
-      - name: kube-rbac-proxy
-        image: gcr.io/kubebuilder/kube-rbac-proxy:v0.8.0
-        args:
-        - "--secure-listen-address=0.0.0.0:8443"
-        - "--upstream=http://127.0.0.1:8080/"
-        - "--logtostderr=true"
-        - "--v=10"
-        ports:
-        - containerPort: 8443
-          name: https
       - name: manager
         args:
         - "--health-probe-bind-address=:8081"
-        - "--metrics-bind-address=127.0.0.1:8080"
+        - "--metrics-bind-address=:8443"
         - "--leader-elect"
         - "--machine-downward-api-label=root-machine-namespace=metadata.labels['downward-api.machinepoollet.ironcore.dev/root-machine-namespace']"
         - "--machine-downward-api-label=root-machine-name=metadata.labels['downward-api.machinepoollet.ironcore.dev/root-machine-name']"

config/volumepoollet-broker/broker-rbac/kustomization.yaml

@@ -10,10 +10,13 @@ resources: # All RBAC will be applied under this service account in
   - cluster_role_binding.yaml
   - leader_election_role.yaml
   - leader_election_role_binding.yaml
-  # Comment the following 4 lines if you want to disable
-  # the auth proxy (https://github.com/brancz/kube-rbac-proxy)
-  # which protects your /metrics endpoint.
-  - auth_proxy_service.yaml
-  - auth_proxy_role.yaml
-  - auth_proxy_role_binding.yaml
-  - auth_proxy_client_clusterrole.yaml
+  # The following RBAC configurations are used to protect
+  # the metrics endpoint with authn/authz. These configurations
+  # ensure that only authorized users and service accounts
+  # can access the metrics endpoint. Comment the following
+  # permissions if you want to disable this protection.
+  # More info: https://book.kubebuilder.io/reference/metrics.html
+  - metrics_service.yaml
+  - metrics_auth_role.yaml
+  - metrics_auth_role_binding.yaml
+  - metrics_reader_role.yaml

config/controller/rbac/auth_proxy_role.yaml → config/volumepoollet-broker/broker-rbac/metrics_auth_role.yaml

@@ -1,7 +1,7 @@
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
-  name: proxy-role
+  name: metrics-auth-role
 rules:
 - apiGroups:
   - authentication.k8s.io

config/volumepoollet-broker/broker-rbac/auth_proxy_role_binding.yaml → config/volumepoollet-broker/broker-rbac/metrics_auth_role_binding.yaml

@@ -1,11 +1,11 @@
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
-  name: proxy-rolebinding
+  name: metrics-auth-rolebinding
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: ClusterRole
-  name: proxy-role
+  name: metrics-auth-role
 subjects:
 - kind: ServiceAccount
   name: controller-manager

config/controller/rbac/auth_proxy_service.yaml → config/volumepoollet-broker/broker-rbac/metrics_service.yaml

@@ -9,6 +9,7 @@ spec:
   ports:
   - name: https
     port: 8443
-    targetPort: https
+    protocol: TCP 
+    targetPort: 8443
   selector:
     control-plane: controller-manager

config/volumepoollet-broker/default/kustomization.yaml

@@ -21,7 +21,7 @@ resources:
 #- ../prometheus
 
 patchesStrategicMerge:
-  - manager_auth_proxy_patch.yaml
+  - manager_metrics_patch.yaml
 
   # Mount the controller config file for loading manager configurations
   # through a ComponentConfig type