diff --git a/cmd/ironcore-controller-manager/main.go b/cmd/ironcore-controller-manager/main.go index ee56f0044..a39999df3 100644 --- a/cmd/ironcore-controller-manager/main.go +++ b/cmd/ironcore-controller-manager/main.go @@ -8,6 +8,7 @@ import ( "flag" "fmt" "os" + "path/filepath" "time" corev1alpha1 "github.com/ironcore-dev/ironcore/api/core/v1alpha1" @@ -29,6 +30,7 @@ import ( quotaevaluatorironcore "github.com/ironcore-dev/ironcore/internal/quota/evaluator/ironcore" "github.com/ironcore-dev/ironcore/utils/quota" "k8s.io/utils/lru" + "sigs.k8s.io/controller-runtime/pkg/certwatcher" "sigs.k8s.io/controller-runtime/pkg/metrics/filters" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" @@ -103,6 +105,7 @@ func init() { func main() { var metricsAddr string var secureMetrics bool + var metricsCertPath, metricsCertName, metricsCertKey string var enableHTTP2 bool var enableLeaderElection bool var probeAddr string @@ -115,8 +118,12 @@ func main() { "Use :8443 for HTTPS or :8080 for HTTP, or leave as 0 to disable the metrics service.") flag.BoolVar(&secureMetrics, "metrics-secure", true, "If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.") + flag.StringVar(&metricsCertPath, "metrics-cert-path", "", + "The directory that contains the metrics server certificate.") + flag.StringVar(&metricsCertName, "metrics-cert-name", "tls.crt", "The name of the metrics server certificate file.") + flag.StringVar(&metricsCertKey, "metrics-cert-key", "tls.key", "The name of the metrics server key file.") flag.BoolVar(&enableHTTP2, "enable-http2", false, - "If set, HTTP/2 will be enabled for the metrics and webhook servers") + "If set, HTTP/2 will be enabled for the metrics servers") flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") flag.BoolVar(&enableLeaderElection, "leader-elect", false, "Enable leader election for controller manager. "+ @@ -191,7 +198,7 @@ func main() { tlsOpts = append(tlsOpts, disableHTTP2) } - // Metrics endpoint is enabled in 'config/default/kustomization.yaml'. The Metrics options configure the server. + // Metrics endpoint is enabled in 'config/controller/default/kustomization.yaml'. The Metrics options configure the server. // More info: // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.19.1/pkg/metrics/server // - https://book.kubebuilder.io/reference/metrics.html @@ -207,10 +214,37 @@ func main() { // can access the metrics endpoint. The RBAC are configured in 'config/controller/rbac/kustomization.yaml'. More info: // https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.19.1/pkg/metrics/filters#WithAuthenticationAndAuthorization metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization + } + + // If the certificate is not specified, controller-runtime will automatically + // generate self-signed certificates for the metrics server. While convenient for development and testing, + // this setup is not recommended for production. + // + // TODO(user): If you enable certManager, uncomment the following lines: + // - [METRICS-WITH-CERTS] at config/controller/default/kustomization.yaml to generate and use certificates + // managed by cert-manager for the metrics server. + // - [PROMETHEUS-WITH-CERTS] at config/controller/prometheus/kustomization.yaml for TLS certification. + + // Create watchers for metrics certificates + var metricsCertWatcher *certwatcher.CertWatcher + + if len(metricsCertPath) > 0 { + setupLog.Info("Initializing metrics certificate watcher using provided certificates", + "metrics-cert-path", metricsCertPath, "metrics-cert-name", metricsCertName, "metrics-cert-key", metricsCertKey) + + var err error + metricsCertWatcher, err = certwatcher.New( + filepath.Join(metricsCertPath, metricsCertName), + filepath.Join(metricsCertPath, metricsCertKey), + ) + if err != nil { + setupLog.Error(err, "to initialize metrics certificate watcher", "error", err) + os.Exit(1) + } - // TODO(user): If CertDir, CertName, and KeyName are not specified, controller-runtime will automatically - // generate self-signed certificates for the metrics server. While convenient for development and testing, - // this setup is not recommended for production. + metricsServerOptions.TLSOpts = append(metricsServerOptions.TLSOpts, func(config *tls.Config) { + config.GetCertificate = metricsCertWatcher.GetCertificate + }) } mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ diff --git a/config/bucketpoollet-broker/default/cert_metrics_manager_patch.yaml b/config/bucketpoollet-broker/default/cert_metrics_manager_patch.yaml new file mode 100644 index 000000000..322165cd3 --- /dev/null +++ b/config/bucketpoollet-broker/default/cert_metrics_manager_patch.yaml @@ -0,0 +1,29 @@ +# This patch adds the args, volumes, and ports to allow the manager to use the metrics-server certs. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system +spec: + template: + spec: + containers: + - name: manager + volumeMounts: + - mountPath: /tmp/k8s-metrics-server/metrics-certs + name: metrics-certs + readOnly: true + args: + - "--metrics-cert-path=/tmp/k8s-metrics-server/metrics-certs" + volumes: + - name: metrics-certs + secret: + secretName: metrics-server-cert + optional: false + items: + - key: ca.crt + path: ca.crt + - key: tls.crt + path: tls.crt + - key: tls.key + path: tls.key diff --git a/config/bucketpoollet-broker/default/kustomization.yaml b/config/bucketpoollet-broker/default/kustomization.yaml index 68b42b941..6e7737bad 100644 --- a/config/bucketpoollet-broker/default/kustomization.yaml +++ b/config/bucketpoollet-broker/default/kustomization.yaml @@ -13,101 +13,147 @@ namePrefix: bucketpoollet- # someName: someValue resources: - - ../broker-rbac - - ../manager -# - ../webhook -# - ../certmanager +- ../broker-rbac +- ../manager +# - ../webhook +# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. 'WEBHOOK' components are required. +# - ../certmanager # [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. -#- ../prometheus +# - ../prometheus patchesStrategicMerge: - - manager_metrics_patch.yaml +# [METRICS] The following patch will enable the metrics endpoint using HTTPS and the port :8443. +# More info: https://book.kubebuilder.io/reference/metrics +- manager_metrics_patch.yaml + +# Uncomment the patches line if you enable Metrics and CertManager +# [METRICS-WITH-CERTS] To enable metrics protected with certManager, uncomment the following line. +# This patch will protect the metrics with certManager self-signed certs. +# - cert_metrics_manager_patch.yaml - # Mount the controller config file for loading manager configurations - # through a ComponentConfig type - #- manager_config_patch.yaml +# Mount the controller config file for loading manager configurations +# through a ComponentConfig type +#- manager_config_patch.yaml - #- manager_webhook_patch.yaml - #- webhookcainjection_patch.yaml +#- manager_webhook_patch.yaml +#- webhookcainjection_patch.yaml -#replacements: -# - source: -# kind: Certificate -# group: cert-manager.io -# version: v1 -# name: serving-cert -# fieldPath: .metadata.namespace -# targets: -# - select: -# kind: ValidatingWebhookConfiguration -# fieldPaths: -# - .metadata.annotations.[cert-manager.io/inject-ca-from] -# options: -# delimiter: '/' -# index: 0 -# create: true -# - select: -# kind: MutatingWebhookConfiguration -# fieldPaths: -# - .metadata.annotations.[cert-manager.io/inject-ca-from] -# options: -# delimiter: '/' -# index: 0 -# create: true -# - source: -# kind: Certificate -# group: cert-manager.io -# version: v1 -# name: serving-cert -# fieldPath: .metadata.name -# targets: -# - select: -# kind: ValidatingWebhookConfiguration -# fieldPaths: -# - .metadata.annotations.[cert-manager.io/inject-ca-from] -# options: -# delimiter: '/' -# index: 1 -# create: true -# - select: -# kind: MutatingWebhookConfiguration -# fieldPaths: -# - .metadata.annotations.[cert-manager.io/inject-ca-from] -# options: -# delimiter: '/' -# index: 1 -# create: true -# - source: -# kind: Service -# version: v1 -# name: webhook-service -# fieldPath: .metadata.name -# targets: -# - select: -# kind: Certificate -# group: cert-manager.io -# version: v1 -# fieldPaths: -# - .spec.dnsNames.0 -# - .spec.dnsNames.1 -# options: -# delimiter: '.' -# index: 0 -# create: true -# - source: -# kind: Service -# version: v1 -# name: webhook-service -# fieldPath: .metadata.namespace -# targets: -# - select: -# kind: Certificate -# group: cert-manager.io -# version: v1 -# fieldPaths: -# - .spec.dnsNames.0 -# - .spec.dnsNames.1 -# options: -# delimiter: '.' -# index: 1 -# create: true +# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix. +# Uncomment the following replacements to add the cert-manager CA injection annotations +# replacements: +# - source: # Uncomment the following block to enable certificates for metrics +# kind: Service +# version: v1 +# name: controller-manager-metrics-service +# fieldPath: metadata.name +# targets: +# - select: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: metrics-certs +# fieldPaths: +# - spec.dnsNames.0 +# - spec.dnsNames.1 +# options: +# delimiter: '.' +# index: 0 +# create: true +# - source: +# kind: Service +# version: v1 +# name: controller-manager-metrics-service +# fieldPath: metadata.namespace +# targets: +# - select: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: metrics-certs +# fieldPaths: +# - spec.dnsNames.0 +# - spec.dnsNames.1 +# options: +# delimiter: '.' +# index: 1 +# create: true +# - source: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: serving-cert +# fieldPath: .metadata.namespace +# targets: +# - select: +# kind: ValidatingWebhookConfiguration +# fieldPaths: +# - .metadata.annotations.[cert-manager.io/inject-ca-from] +# options: +# delimiter: '/' +# index: 0 +# create: true +# - select: +# kind: MutatingWebhookConfiguration +# fieldPaths: +# - .metadata.annotations.[cert-manager.io/inject-ca-from] +# options: +# delimiter: '/' +# index: 0 +# create: true +# - source: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: serving-cert +# fieldPath: .metadata.name +# targets: +# - select: +# kind: ValidatingWebhookConfiguration +# fieldPaths: +# - .metadata.annotations.[cert-manager.io/inject-ca-from] +# options: +# delimiter: '/' +# index: 1 +# create: true +# - select: +# kind: MutatingWebhookConfiguration +# fieldPaths: +# - .metadata.annotations.[cert-manager.io/inject-ca-from] +# options: +# delimiter: '/' +# index: 1 +# create: true +# - source: +# kind: Service +# version: v1 +# name: webhook-service +# fieldPath: .metadata.name +# targets: +# - select: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# fieldPaths: +# - .spec.dnsNames.0 +# - .spec.dnsNames.1 +# options: +# delimiter: '.' +# index: 0 +# create: true +# - source: +# kind: Service +# version: v1 +# name: webhook-service +# fieldPath: .metadata.namespace +# targets: +# - select: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# fieldPaths: +# - .spec.dnsNames.0 +# - .spec.dnsNames.1 +# options: +# delimiter: '.' +# index: 1 +# create: true diff --git a/config/bucketpoollet-broker/prometheus/kustomization.yaml b/config/bucketpoollet-broker/prometheus/kustomization.yaml index ed137168a..f64ffe8e4 100644 --- a/config/bucketpoollet-broker/prometheus/kustomization.yaml +++ b/config/bucketpoollet-broker/prometheus/kustomization.yaml @@ -1,2 +1,11 @@ resources: - monitor.yaml + +# [PROMETHEUS-WITH-CERTS] The following patch configures the ServiceMonitor in ../prometheus +# to securely reference certificates created and managed by cert-manager. +# Additionally, ensure that you uncomment the [METRICS-WITH-CERTS] patch under config/bucketpoollet-broker/default/kustomization.yaml +# to mount the "metrics-server-cert" secret in the Manager Deployment. +#patches: +# - path: monitor_tls_patch.yaml +# target: +# kind: ServiceMonitor diff --git a/config/bucketpoollet-broker/prometheus/monitor.yaml b/config/bucketpoollet-broker/prometheus/monitor.yaml index d19136ae7..329e110d9 100644 --- a/config/bucketpoollet-broker/prometheus/monitor.yaml +++ b/config/bucketpoollet-broker/prometheus/monitor.yaml @@ -10,10 +10,15 @@ metadata: spec: endpoints: - path: /metrics - port: https + port: https # Ensure this is the name of the port that exposes HTTPS metrics scheme: https bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token tlsConfig: + # TODO(user): The option insecureSkipVerify: true is not recommended for production since it disables + # certificate verification, exposing the system to potential man-in-the-middle attacks. + # For production environments, it is recommended to use cert-manager for automatic TLS certificate management. + # To apply this configuration, enable cert-manager and use the patch located at config/bucketpoollet-broker/prometheus/monitor_tls_patch.yaml, + # which securely references the certificate from the 'metrics-server-cert' secret. insecureSkipVerify: true selector: matchLabels: diff --git a/config/bucketpoollet-broker/prometheus/monitor_tls_patch.yaml b/config/bucketpoollet-broker/prometheus/monitor_tls_patch.yaml new file mode 100644 index 000000000..e824dd0ff --- /dev/null +++ b/config/bucketpoollet-broker/prometheus/monitor_tls_patch.yaml @@ -0,0 +1,22 @@ +# Patch for Prometheus ServiceMonitor to enable secure TLS configuration +# using certificates managed by cert-manager +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: controller-manager-metrics-monitor + namespace: system +spec: + endpoints: + - tlsConfig: + insecureSkipVerify: false + ca: + secret: + name: metrics-server-cert + key: ca.crt + cert: + secret: + name: metrics-server-cert + key: tls.crt + keySecret: + name: metrics-server-cert + key: tls.key diff --git a/config/controller/default/cert_metrics_manager_patch.yaml b/config/controller/default/cert_metrics_manager_patch.yaml new file mode 100644 index 000000000..322165cd3 --- /dev/null +++ b/config/controller/default/cert_metrics_manager_patch.yaml @@ -0,0 +1,29 @@ +# This patch adds the args, volumes, and ports to allow the manager to use the metrics-server certs. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system +spec: + template: + spec: + containers: + - name: manager + volumeMounts: + - mountPath: /tmp/k8s-metrics-server/metrics-certs + name: metrics-certs + readOnly: true + args: + - "--metrics-cert-path=/tmp/k8s-metrics-server/metrics-certs" + volumes: + - name: metrics-certs + secret: + secretName: metrics-server-cert + optional: false + items: + - key: ca.crt + path: ca.crt + - key: tls.crt + path: tls.crt + - key: tls.key + path: tls.key diff --git a/config/controller/default/kustomization.yaml b/config/controller/default/kustomization.yaml index 3401a9e16..83f875b3a 100644 --- a/config/controller/default/kustomization.yaml +++ b/config/controller/default/kustomization.yaml @@ -13,101 +13,147 @@ namePrefix: ironcore- # someName: someValue resources: - - ../rbac - - ../manager - - ../webhook - - ../certmanager +- ../rbac +- ../manager +- ../webhook +# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. 'WEBHOOK' components are required. +- ../certmanager # [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. -#- ../prometheus +# - ../prometheus patchesStrategicMerge: - - manager_metrics_patch.yaml +# [METRICS] The following patch will enable the metrics endpoint using HTTPS and the port :8443. +# More info: https://book.kubebuilder.io/reference/metrics +- manager_metrics_patch.yaml - # Mount the controller config file for loading manager configurations - # through a ComponentConfig type - #- manager_config_patch.yaml +# Uncomment the patches line if you enable Metrics and CertManager +# [METRICS-WITH-CERTS] To enable metrics protected with certManager, uncomment the following line. +# This patch will protect the metrics with certManager self-signed certs. +# - cert_metrics_manager_patch.yaml - - manager_webhook_patch.yaml - # - webhookcainjection_patch.yaml +# Mount the controller config file for loading manager configurations +# through a ComponentConfig type +#- manager_config_patch.yaml +- manager_webhook_patch.yaml +# - webhookcainjection_patch.yaml + +# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix. +# Uncomment the following replacements to add the cert-manager CA injection annotations replacements: - - source: - kind: Certificate - group: cert-manager.io - version: v1 - name: serving-cert - fieldPath: .metadata.namespace - targets: - - select: - kind: ValidatingWebhookConfiguration - fieldPaths: - - .metadata.annotations.[cert-manager.io/inject-ca-from] - options: - delimiter: '/' - index: 0 - create: true - - select: - kind: MutatingWebhookConfiguration - fieldPaths: - - .metadata.annotations.[cert-manager.io/inject-ca-from] - options: - delimiter: '/' - index: 0 - create: true - - source: - kind: Certificate - group: cert-manager.io - version: v1 - name: serving-cert - fieldPath: .metadata.name - targets: - - select: - kind: ValidatingWebhookConfiguration - fieldPaths: - - .metadata.annotations.[cert-manager.io/inject-ca-from] - options: - delimiter: '/' - index: 1 - create: true - - select: - kind: MutatingWebhookConfiguration - fieldPaths: - - .metadata.annotations.[cert-manager.io/inject-ca-from] - options: - delimiter: '/' - index: 1 - create: true - - source: - kind: Service - version: v1 - name: webhook-service - fieldPath: .metadata.name - targets: - - select: - kind: Certificate - group: cert-manager.io - version: v1 - fieldPaths: - - .spec.dnsNames.0 - - .spec.dnsNames.1 - options: - delimiter: '.' - index: 0 - create: true - - source: - kind: Service - version: v1 - name: webhook-service - fieldPath: .metadata.namespace - targets: - - select: - kind: Certificate - group: cert-manager.io - version: v1 - fieldPaths: - - .spec.dnsNames.0 - - .spec.dnsNames.1 - options: - delimiter: '.' - index: 1 - create: true +# - source: # Uncomment the following block to enable certificates for metrics +# kind: Service +# version: v1 +# name: controller-manager-metrics-service +# fieldPath: metadata.name +# targets: +# - select: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: metrics-certs +# fieldPaths: +# - spec.dnsNames.0 +# - spec.dnsNames.1 +# options: +# delimiter: '.' +# index: 0 +# create: true +# - source: +# kind: Service +# version: v1 +# name: controller-manager-metrics-service +# fieldPath: metadata.namespace +# targets: +# - select: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: metrics-certs +# fieldPaths: +# - spec.dnsNames.0 +# - spec.dnsNames.1 +# options: +# delimiter: '.' +# index: 1 +# create: true +- source: + kind: Certificate + group: cert-manager.io + version: v1 + name: serving-cert + fieldPath: .metadata.namespace + targets: + - select: + kind: ValidatingWebhookConfiguration + fieldPaths: + - .metadata.annotations.[cert-manager.io/inject-ca-from] + options: + delimiter: '/' + index: 0 + create: true + - select: + kind: MutatingWebhookConfiguration + fieldPaths: + - .metadata.annotations.[cert-manager.io/inject-ca-from] + options: + delimiter: '/' + index: 0 + create: true +- source: + kind: Certificate + group: cert-manager.io + version: v1 + name: serving-cert + fieldPath: .metadata.name + targets: + - select: + kind: ValidatingWebhookConfiguration + fieldPaths: + - .metadata.annotations.[cert-manager.io/inject-ca-from] + options: + delimiter: '/' + index: 1 + create: true + - select: + kind: MutatingWebhookConfiguration + fieldPaths: + - .metadata.annotations.[cert-manager.io/inject-ca-from] + options: + delimiter: '/' + index: 1 + create: true +- source: + kind: Service + version: v1 + name: webhook-service + fieldPath: .metadata.name + targets: + - select: + kind: Certificate + group: cert-manager.io + version: v1 + fieldPaths: + - .spec.dnsNames.0 + - .spec.dnsNames.1 + options: + delimiter: '.' + index: 0 + create: true +- source: + kind: Service + version: v1 + name: webhook-service + fieldPath: .metadata.namespace + targets: + - select: + kind: Certificate + group: cert-manager.io + version: v1 + fieldPaths: + - .spec.dnsNames.0 + - .spec.dnsNames.1 + options: + delimiter: '.' + index: 1 + create: true diff --git a/config/controller/prometheus/kustomization.yaml b/config/controller/prometheus/kustomization.yaml index ed137168a..01616c9eb 100644 --- a/config/controller/prometheus/kustomization.yaml +++ b/config/controller/prometheus/kustomization.yaml @@ -1,2 +1,11 @@ resources: - monitor.yaml + +# [PROMETHEUS-WITH-CERTS] The following patch configures the ServiceMonitor in ../prometheus +# to securely reference certificates created and managed by cert-manager. +# Additionally, ensure that you uncomment the [METRICS-WITH-CERTS] patch under config/controller/default/kustomization.yaml +# to mount the "metrics-server-cert" secret in the Manager Deployment. +#patches: +# - path: monitor_tls_patch.yaml +# target: +# kind: ServiceMonitor diff --git a/config/controller/prometheus/monitor.yaml b/config/controller/prometheus/monitor.yaml index d19136ae7..431f3e7c3 100644 --- a/config/controller/prometheus/monitor.yaml +++ b/config/controller/prometheus/monitor.yaml @@ -10,10 +10,15 @@ metadata: spec: endpoints: - path: /metrics - port: https + port: https # Ensure this is the name of the port that exposes HTTPS metrics scheme: https bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token tlsConfig: + # TODO(user): The option insecureSkipVerify: true is not recommended for production since it disables + # certificate verification, exposing the system to potential man-in-the-middle attacks. + # For production environments, it is recommended to use cert-manager for automatic TLS certificate management. + # To apply this configuration, enable cert-manager and use the patch located at config/controller/prometheus/monitor_tls_patch.yaml, + # which securely references the certificate from the 'metrics-server-cert' secret. insecureSkipVerify: true selector: matchLabels: diff --git a/config/controller/prometheus/monitor_tls_patch.yaml b/config/controller/prometheus/monitor_tls_patch.yaml new file mode 100644 index 000000000..e824dd0ff --- /dev/null +++ b/config/controller/prometheus/monitor_tls_patch.yaml @@ -0,0 +1,22 @@ +# Patch for Prometheus ServiceMonitor to enable secure TLS configuration +# using certificates managed by cert-manager +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: controller-manager-metrics-monitor + namespace: system +spec: + endpoints: + - tlsConfig: + insecureSkipVerify: false + ca: + secret: + name: metrics-server-cert + key: ca.crt + cert: + secret: + name: metrics-server-cert + key: tls.crt + keySecret: + name: metrics-server-cert + key: tls.key diff --git a/config/machinepoollet-broker/default/cert_metrics_manager_patch.yaml b/config/machinepoollet-broker/default/cert_metrics_manager_patch.yaml new file mode 100644 index 000000000..322165cd3 --- /dev/null +++ b/config/machinepoollet-broker/default/cert_metrics_manager_patch.yaml @@ -0,0 +1,29 @@ +# This patch adds the args, volumes, and ports to allow the manager to use the metrics-server certs. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system +spec: + template: + spec: + containers: + - name: manager + volumeMounts: + - mountPath: /tmp/k8s-metrics-server/metrics-certs + name: metrics-certs + readOnly: true + args: + - "--metrics-cert-path=/tmp/k8s-metrics-server/metrics-certs" + volumes: + - name: metrics-certs + secret: + secretName: metrics-server-cert + optional: false + items: + - key: ca.crt + path: ca.crt + - key: tls.crt + path: tls.crt + - key: tls.key + path: tls.key diff --git a/config/machinepoollet-broker/default/kustomization.yaml b/config/machinepoollet-broker/default/kustomization.yaml index 6d6ff2454..675de1e3a 100644 --- a/config/machinepoollet-broker/default/kustomization.yaml +++ b/config/machinepoollet-broker/default/kustomization.yaml @@ -15,14 +15,23 @@ namePrefix: machinepoollet- resources: - ../broker-rbac - ../manager -# - ../webhook -# - ../certmanager +# - ../webhook +# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. 'WEBHOOK' components are required. +# - ../certmanager # [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. #- ../prometheus patchesStrategicMerge: +# [METRICS] The following patch will enable the metrics endpoint using HTTPS and the port :8443. +# More info: https://book.kubebuilder.io/reference/metrics - manager_metrics_patch.yaml +# Uncomment the patches line if you enable Metrics and CertManager +# [METRICS-WITH-CERTS] To enable metrics protected with certManager, uncomment the following line. +# This patch will protect the metrics with certManager self-signed certs. +# - cert_metrics_manager_patch.yaml + + # Mount the controller config file for loading manager configurations # through a ComponentConfig type #- manager_config_patch.yaml @@ -30,7 +39,45 @@ patchesStrategicMerge: #- manager_webhook_patch.yaml #- webhookcainjection_patch.yaml +# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix. +# Uncomment the following replacements to add the cert-manager CA injection annotations replacements: +# - source: # Uncomment the following block to enable certificates for metrics +# kind: Service +# version: v1 +# name: controller-manager-metrics-service +# fieldPath: metadata.name +# targets: +# - select: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: metrics-certs +# fieldPaths: +# - spec.dnsNames.0 +# - spec.dnsNames.1 +# options: +# delimiter: '.' +# index: 0 +# create: true +# - source: +# kind: Service +# version: v1 +# name: controller-manager-metrics-service +# fieldPath: metadata.namespace +# targets: +# - select: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: metrics-certs +# fieldPaths: +# - spec.dnsNames.0 +# - spec.dnsNames.1 +# options: +# delimiter: '.' +# index: 1 +# create: true - source: kind: Service group: "" diff --git a/config/machinepoollet-broker/prometheus/kustomization.yaml b/config/machinepoollet-broker/prometheus/kustomization.yaml index ed137168a..5f84e634d 100644 --- a/config/machinepoollet-broker/prometheus/kustomization.yaml +++ b/config/machinepoollet-broker/prometheus/kustomization.yaml @@ -1,2 +1,11 @@ resources: - monitor.yaml + +# [PROMETHEUS-WITH-CERTS] The following patch configures the ServiceMonitor in ../prometheus +# to securely reference certificates created and managed by cert-manager. +# Additionally, ensure that you uncomment the [METRICS-WITH-CERTS] patch under config/machinepoollet-broker/default/kustomization.yaml +# to mount the "metrics-server-cert" secret in the Manager Deployment. +#patches: +# - path: monitor_tls_patch.yaml +# target: +# kind: ServiceMonitor diff --git a/config/machinepoollet-broker/prometheus/monitor.yaml b/config/machinepoollet-broker/prometheus/monitor.yaml index d19136ae7..c663b7335 100644 --- a/config/machinepoollet-broker/prometheus/monitor.yaml +++ b/config/machinepoollet-broker/prometheus/monitor.yaml @@ -10,10 +10,15 @@ metadata: spec: endpoints: - path: /metrics - port: https + port: https # Ensure this is the name of the port that exposes HTTPS metrics scheme: https bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token tlsConfig: + # TODO(user): The option insecureSkipVerify: true is not recommended for production since it disables + # certificate verification, exposing the system to potential man-in-the-middle attacks. + # For production environments, it is recommended to use cert-manager for automatic TLS certificate management. + # To apply this configuration, enable cert-manager and use the patch located at config/machinepoollet-broker/prometheus/monitor_tls_patch.yaml, + # which securely references the certificate from the 'metrics-server-cert' secret. insecureSkipVerify: true selector: matchLabels: diff --git a/config/machinepoollet-broker/prometheus/monitor_tls_patch.yaml b/config/machinepoollet-broker/prometheus/monitor_tls_patch.yaml new file mode 100644 index 000000000..e824dd0ff --- /dev/null +++ b/config/machinepoollet-broker/prometheus/monitor_tls_patch.yaml @@ -0,0 +1,22 @@ +# Patch for Prometheus ServiceMonitor to enable secure TLS configuration +# using certificates managed by cert-manager +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: controller-manager-metrics-monitor + namespace: system +spec: + endpoints: + - tlsConfig: + insecureSkipVerify: false + ca: + secret: + name: metrics-server-cert + key: ca.crt + cert: + secret: + name: metrics-server-cert + key: tls.crt + keySecret: + name: metrics-server-cert + key: tls.key diff --git a/config/volumepoollet-broker/default/cert_metrics_manager_patch.yaml b/config/volumepoollet-broker/default/cert_metrics_manager_patch.yaml new file mode 100644 index 000000000..322165cd3 --- /dev/null +++ b/config/volumepoollet-broker/default/cert_metrics_manager_patch.yaml @@ -0,0 +1,29 @@ +# This patch adds the args, volumes, and ports to allow the manager to use the metrics-server certs. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system +spec: + template: + spec: + containers: + - name: manager + volumeMounts: + - mountPath: /tmp/k8s-metrics-server/metrics-certs + name: metrics-certs + readOnly: true + args: + - "--metrics-cert-path=/tmp/k8s-metrics-server/metrics-certs" + volumes: + - name: metrics-certs + secret: + secretName: metrics-server-cert + optional: false + items: + - key: ca.crt + path: ca.crt + - key: tls.crt + path: tls.crt + - key: tls.key + path: tls.key diff --git a/config/volumepoollet-broker/default/kustomization.yaml b/config/volumepoollet-broker/default/kustomization.yaml index 09d5e6b20..89f76b2e9 100644 --- a/config/volumepoollet-broker/default/kustomization.yaml +++ b/config/volumepoollet-broker/default/kustomization.yaml @@ -13,101 +13,147 @@ namePrefix: volumepoollet- # someName: someValue resources: - - ../broker-rbac - - ../manager -# - ../webhook -# - ../certmanager +- ../broker-rbac +- ../manager +# - ../webhook +# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. 'WEBHOOK' components are required. +# - ../certmanager # [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. -#- ../prometheus +# - ../prometheus patchesStrategicMerge: - - manager_metrics_patch.yaml +# [METRICS] The following patch will enable the metrics endpoint using HTTPS and the port :8443. +# More info: https://book.kubebuilder.io/reference/metrics +- manager_metrics_patch.yaml + +# Uncomment the patches line if you enable Metrics and CertManager +# [METRICS-WITH-CERTS] To enable metrics protected with certManager, uncomment the following line. +# This patch will protect the metrics with certManager self-signed certs. +# - cert_metrics_manager_patch.yaml - # Mount the controller config file for loading manager configurations - # through a ComponentConfig type - #- manager_config_patch.yaml +# Mount the controller config file for loading manager configurations +# through a ComponentConfig type +# - manager_config_patch.yaml - #- manager_webhook_patch.yaml - #- webhookcainjection_patch.yaml +# - manager_webhook_patch.yaml +# - webhookcainjection_patch.yaml -#replacements: -# - source: -# kind: Certificate -# group: cert-manager.io -# version: v1 -# name: serving-cert -# fieldPath: .metadata.namespace -# targets: -# - select: -# kind: ValidatingWebhookConfiguration -# fieldPaths: -# - .metadata.annotations.[cert-manager.io/inject-ca-from] -# options: -# delimiter: '/' -# index: 0 -# create: true -# - select: -# kind: MutatingWebhookConfiguration -# fieldPaths: -# - .metadata.annotations.[cert-manager.io/inject-ca-from] -# options: -# delimiter: '/' -# index: 0 -# create: true -# - source: -# kind: Certificate -# group: cert-manager.io -# version: v1 -# name: serving-cert -# fieldPath: .metadata.name -# targets: -# - select: -# kind: ValidatingWebhookConfiguration -# fieldPaths: -# - .metadata.annotations.[cert-manager.io/inject-ca-from] -# options: -# delimiter: '/' -# index: 1 -# create: true -# - select: -# kind: MutatingWebhookConfiguration -# fieldPaths: -# - .metadata.annotations.[cert-manager.io/inject-ca-from] -# options: -# delimiter: '/' -# index: 1 -# create: true -# - source: -# kind: Service -# version: v1 -# name: webhook-service -# fieldPath: .metadata.name -# targets: -# - select: -# kind: Certificate -# group: cert-manager.io -# version: v1 -# fieldPaths: -# - .spec.dnsNames.0 -# - .spec.dnsNames.1 -# options: -# delimiter: '.' -# index: 0 -# create: true -# - source: -# kind: Service -# version: v1 -# name: webhook-service -# fieldPath: .metadata.namespace -# targets: -# - select: -# kind: Certificate -# group: cert-manager.io -# version: v1 -# fieldPaths: -# - .spec.dnsNames.0 -# - .spec.dnsNames.1 -# options: -# delimiter: '.' -# index: 1 -# create: true +# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix. +# Uncomment the following replacements to add the cert-manager CA injection annotations +# replacements: +# - source: # Uncomment the following block to enable certificates for metrics +# kind: Service +# version: v1 +# name: controller-manager-metrics-service +# fieldPath: metadata.name +# targets: +# - select: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: metrics-certs +# fieldPaths: +# - spec.dnsNames.0 +# - spec.dnsNames.1 +# options: +# delimiter: '.' +# index: 0 +# create: true +# - source: +# kind: Service +# version: v1 +# name: controller-manager-metrics-service +# fieldPath: metadata.namespace +# targets: +# - select: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: metrics-certs +# fieldPaths: +# - spec.dnsNames.0 +# - spec.dnsNames.1 +# options: +# delimiter: '.' +# index: 1 +# create: true +# - source: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: serving-cert +# fieldPath: .metadata.namespace +# targets: +# - select: +# kind: ValidatingWebhookConfiguration +# fieldPaths: +# - .metadata.annotations.[cert-manager.io/inject-ca-from] +# options: +# delimiter: '/' +# index: 0 +# create: true +# - select: +# kind: MutatingWebhookConfiguration +# fieldPaths: +# - .metadata.annotations.[cert-manager.io/inject-ca-from] +# options: +# delimiter: '/' +# index: 0 +# create: true +# - source: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: serving-cert +# fieldPath: .metadata.name +# targets: +# - select: +# kind: ValidatingWebhookConfiguration +# fieldPaths: +# - .metadata.annotations.[cert-manager.io/inject-ca-from] +# options: +# delimiter: '/' +# index: 1 +# create: true +# - select: +# kind: MutatingWebhookConfiguration +# fieldPaths: +# - .metadata.annotations.[cert-manager.io/inject-ca-from] +# options: +# delimiter: '/' +# index: 1 +# create: true +# - source: +# kind: Service +# version: v1 +# name: webhook-service +# fieldPath: .metadata.name +# targets: +# - select: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# fieldPaths: +# - .spec.dnsNames.0 +# - .spec.dnsNames.1 +# options: +# delimiter: '.' +# index: 0 +# create: true +# - source: +# kind: Service +# version: v1 +# name: webhook-service +# fieldPath: .metadata.namespace +# targets: +# - select: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# fieldPaths: +# - .spec.dnsNames.0 +# - .spec.dnsNames.1 +# options: +# delimiter: '.' +# index: 1 +# create: true diff --git a/config/volumepoollet-broker/prometheus/kustomization.yaml b/config/volumepoollet-broker/prometheus/kustomization.yaml index ed137168a..80bbaf02e 100644 --- a/config/volumepoollet-broker/prometheus/kustomization.yaml +++ b/config/volumepoollet-broker/prometheus/kustomization.yaml @@ -1,2 +1,11 @@ resources: - monitor.yaml + +# [PROMETHEUS-WITH-CERTS] The following patch configures the ServiceMonitor in ../prometheus +# to securely reference certificates created and managed by cert-manager. +# Additionally, ensure that you uncomment the [METRICS-WITH-CERTS] patch under config/volumepoollet-broker/default/kustomization.yaml +# to mount the "metrics-server-cert" secret in the Manager Deployment. +#patches: +# - path: monitor_tls_patch.yaml +# target: +# kind: ServiceMonitor diff --git a/config/volumepoollet-broker/prometheus/monitor.yaml b/config/volumepoollet-broker/prometheus/monitor.yaml index d19136ae7..8f9bf8fe9 100644 --- a/config/volumepoollet-broker/prometheus/monitor.yaml +++ b/config/volumepoollet-broker/prometheus/monitor.yaml @@ -10,10 +10,15 @@ metadata: spec: endpoints: - path: /metrics - port: https + port: https # Ensure this is the name of the port that exposes HTTPS metrics scheme: https bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token tlsConfig: + # TODO(user): The option insecureSkipVerify: true is not recommended for production since it disables + # certificate verification, exposing the system to potential man-in-the-middle attacks. + # For production environments, it is recommended to use cert-manager for automatic TLS certificate management. + # To apply this configuration, enable cert-manager and use the patch located at config/volumepoollet-broker/prometheus/monitor_tls_patch.yaml, + # which securely references the certificate from the 'metrics-server-cert' secret. insecureSkipVerify: true selector: matchLabels: diff --git a/config/volumepoollet-broker/prometheus/monitor_tls_patch.yaml b/config/volumepoollet-broker/prometheus/monitor_tls_patch.yaml new file mode 100644 index 000000000..e824dd0ff --- /dev/null +++ b/config/volumepoollet-broker/prometheus/monitor_tls_patch.yaml @@ -0,0 +1,22 @@ +# Patch for Prometheus ServiceMonitor to enable secure TLS configuration +# using certificates managed by cert-manager +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: controller-manager-metrics-monitor + namespace: system +spec: + endpoints: + - tlsConfig: + insecureSkipVerify: false + ca: + secret: + name: metrics-server-cert + key: ca.crt + cert: + secret: + name: metrics-server-cert + key: tls.crt + keySecret: + name: metrics-server-cert + key: tls.key diff --git a/poollet/bucketpoollet/cmd/bucketpoollet/app/app.go b/poollet/bucketpoollet/cmd/bucketpoollet/app/app.go index 843e37d36..3bc821fd2 100644 --- a/poollet/bucketpoollet/cmd/bucketpoollet/app/app.go +++ b/poollet/bucketpoollet/cmd/bucketpoollet/app/app.go @@ -9,6 +9,7 @@ import ( goflag "flag" "fmt" "os" + "path/filepath" "time" "github.com/spf13/cobra" @@ -31,6 +32,7 @@ import ( utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/certwatcher" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" "sigs.k8s.io/controller-runtime/pkg/metrics/filters" @@ -51,6 +53,9 @@ type Options struct { GetConfigOptions config.GetConfigOptions MetricsAddr string SecureMetrics bool + MetricsCertPath string + MetricsCertName string + MetricsCertKey string EnableHTTP2 bool EnableLeaderElection bool LeaderElectionNamespace string @@ -79,8 +84,12 @@ func (o *Options) AddFlags(fs *pflag.FlagSet) { "Use :8443 for HTTPS or :8080 for HTTP, or leave as 0 to disable the metrics service.") fs.BoolVar(&o.SecureMetrics, "metrics-secure", true, "If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.") + fs.StringVar(&o.MetricsCertPath, "metrics-cert-path", "", + "The directory that contains the metrics server certificate.") + fs.StringVar(&o.MetricsCertName, "metrics-cert-name", "tls.crt", "The name of the metrics server certificate file.") + fs.StringVar(&o.MetricsCertKey, "metrics-cert-key", "tls.key", "The name of the metrics server key file.") fs.BoolVar(&o.EnableHTTP2, "enable-http2", false, - "If set, HTTP/2 will be enabled for the metrics and webhook servers") + "If set, HTTP/2 will be enabled for the metrics servers") fs.StringVar(&o.ProbeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") fs.BoolVar(&o.EnableLeaderElection, "leader-elect", false, "Enable leader election for controller manager. "+ @@ -187,7 +196,7 @@ func Run(ctx context.Context, opts Options) error { if !opts.EnableHTTP2 { tlsOpts = append(tlsOpts, disableHTTP2) } - // Metrics endpoint is enabled in 'config/default/kustomization.yaml'. The Metrics options configure the server. + // Metrics endpoint is enabled in 'config/bucketpoollet-broker/default/kustomization.yaml'. The Metrics options configure the server. // More info: // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.19.1/pkg/metrics/server // - https://book.kubebuilder.io/reference/metrics.html @@ -202,9 +211,37 @@ func Run(ctx context.Context, opts Options) error { // can access the metrics endpoint. The RBAC are configured in 'config/bucketpoollet-broker/broker-rbac/kustomization.yaml'. More info: // https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.19.1/pkg/metrics/filters#WithAuthenticationAndAuthorization metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization - // TODO(user): If CertDir, CertName, and KeyName are not specified, controller-runtime will automatically - // generate self-signed certificates for the metrics server. While convenient for development and testing, - // this setup is not recommended for production. + } + + // If the certificate is not specified, controller-runtime will automatically + // generate self-signed certificates for the metrics server. While convenient for development and testing, + // this setup is not recommended for production. + // + // TODO(user): If you enable certManager, uncomment the following lines: + // - [METRICS-WITH-CERTS] at config/bucketpoollet-broker/default/kustomization.yaml to generate and use certificates + // managed by cert-manager for the metrics server. + // - [PROMETHEUS-WITH-CERTS] at config/bucketpoollet-broker/prometheus/kustomization.yaml for TLS certification. + + // Create watchers for metrics certificates + var metricsCertWatcher *certwatcher.CertWatcher + + if len(opts.MetricsCertPath) > 0 { + setupLog.Info("Initializing metrics certificate watcher using provided certificates", + "metrics-cert-path", opts.MetricsCertPath, "metrics-cert-name", opts.MetricsCertName, "metrics-cert-key", opts.MetricsCertKey) + + var err error + metricsCertWatcher, err = certwatcher.New( + filepath.Join(opts.MetricsCertPath, opts.MetricsCertName), + filepath.Join(opts.MetricsCertPath, opts.MetricsCertKey), + ) + if err != nil { + setupLog.Error(err, "to initialize metrics certificate watcher", "error", err) + os.Exit(1) + } + + metricsServerOptions.TLSOpts = append(metricsServerOptions.TLSOpts, func(config *tls.Config) { + config.GetCertificate = metricsCertWatcher.GetCertificate + }) } mgr, err := ctrl.NewManager(cfg, ctrl.Options{ diff --git a/poollet/machinepoollet/cmd/machinepoollet/app/app.go b/poollet/machinepoollet/cmd/machinepoollet/app/app.go index 7c30ae5b6..0fdaeb8c8 100644 --- a/poollet/machinepoollet/cmd/machinepoollet/app/app.go +++ b/poollet/machinepoollet/cmd/machinepoollet/app/app.go @@ -9,6 +9,8 @@ import ( goflag "flag" "fmt" "net" + "os" + "path/filepath" "strconv" "time" @@ -40,6 +42,7 @@ import ( "k8s.io/client-go/rest" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/cache" + "sigs.k8s.io/controller-runtime/pkg/certwatcher" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" @@ -61,6 +64,9 @@ type Options struct { GetConfigOptions config.GetConfigOptions MetricsAddr string SecureMetrics bool + MetricsCertPath string + MetricsCertName string + MetricsCertKey string EnableHTTP2 bool EnableLeaderElection bool LeaderElectionNamespace string @@ -95,8 +101,12 @@ func (o *Options) AddFlags(fs *pflag.FlagSet) { "Use :8443 for HTTPS or :8080 for HTTP, or leave as 0 to disable the metrics service.") fs.BoolVar(&o.SecureMetrics, "metrics-secure", true, "If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.") + fs.StringVar(&o.MetricsCertPath, "metrics-cert-path", "", + "The directory that contains the metrics server certificate.") + fs.StringVar(&o.MetricsCertName, "metrics-cert-name", "tls.crt", "The name of the metrics server certificate file.") + fs.StringVar(&o.MetricsCertKey, "metrics-cert-key", "tls.key", "The name of the metrics server key file.") fs.BoolVar(&o.EnableHTTP2, "enable-http2", false, - "If set, HTTP/2 will be enabled for the metrics and webhook servers") + "If set, HTTP/2 will be enabled for the metrics servers") fs.StringVar(&o.ProbeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") fs.BoolVar(&o.EnableLeaderElection, "leader-elect", false, "Enable leader election for controller manager. "+ @@ -243,7 +253,7 @@ func Run(ctx context.Context, opts Options) error { if !opts.EnableHTTP2 { tlsOpts = append(tlsOpts, disableHTTP2) } - // Metrics endpoint is enabled in 'config/default/kustomization.yaml'. The Metrics options configure the server. + // Metrics endpoint is enabled in 'config/machinepoollet-broker/default/kustomization.yaml'. The Metrics options configure the server. // More info: // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.19.1/pkg/metrics/server // - https://book.kubebuilder.io/reference/metrics.html @@ -258,9 +268,37 @@ func Run(ctx context.Context, opts Options) error { // can access the metrics endpoint. The RBAC are configured in 'config/machinepoollet-broker/broker-rbac/kustomization.yaml'. More info: // https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.19.1/pkg/metrics/filters#WithAuthenticationAndAuthorization metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization - // TODO(user): If CertDir, CertName, and KeyName are not specified, controller-runtime will automatically - // generate self-signed certificates for the metrics server. While convenient for development and testing, - // this setup is not recommended for production. + } + + // If the certificate is not specified, controller-runtime will automatically + // generate self-signed certificates for the metrics server. While convenient for development and testing, + // this setup is not recommended for production. + // + // TODO(user): If you enable certManager, uncomment the following lines: + // - [METRICS-WITH-CERTS] at config/machinepoollet-broker/default/kustomization.yaml to generate and use certificates + // managed by cert-manager for the metrics server. + // - [PROMETHEUS-WITH-CERTS] at config/machinepoollet-broker/prometheus/kustomization.yaml for TLS certification. + + // Create watchers for metrics certificates + var metricsCertWatcher *certwatcher.CertWatcher + + if len(opts.MetricsCertPath) > 0 { + setupLog.Info("Initializing metrics certificate watcher using provided certificates", + "metrics-cert-path", opts.MetricsCertPath, "metrics-cert-name", opts.MetricsCertName, "metrics-cert-key", opts.MetricsCertKey) + + var err error + metricsCertWatcher, err = certwatcher.New( + filepath.Join(opts.MetricsCertPath, opts.MetricsCertName), + filepath.Join(opts.MetricsCertPath, opts.MetricsCertKey), + ) + if err != nil { + setupLog.Error(err, "to initialize metrics certificate watcher", "error", err) + os.Exit(1) + } + + metricsServerOptions.TLSOpts = append(metricsServerOptions.TLSOpts, func(config *tls.Config) { + config.GetCertificate = metricsCertWatcher.GetCertificate + }) } mgr, err := ctrl.NewManager(cfg, ctrl.Options{ diff --git a/poollet/volumepoollet/cmd/volumepoollet/app/app.go b/poollet/volumepoollet/cmd/volumepoollet/app/app.go index eaef2f05c..1f350d939 100644 --- a/poollet/volumepoollet/cmd/volumepoollet/app/app.go +++ b/poollet/volumepoollet/cmd/volumepoollet/app/app.go @@ -9,6 +9,7 @@ import ( goflag "flag" "fmt" "os" + "path/filepath" "time" "github.com/spf13/cobra" @@ -32,6 +33,7 @@ import ( utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/certwatcher" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" "sigs.k8s.io/controller-runtime/pkg/metrics/filters" @@ -52,6 +54,9 @@ type Options struct { GetConfigOptions config.GetConfigOptions MetricsAddr string SecureMetrics bool + MetricsCertPath string + MetricsCertName string + MetricsCertKey string EnableHTTP2 bool EnableLeaderElection bool LeaderElectionNamespace string @@ -80,8 +85,12 @@ func (o *Options) AddFlags(fs *pflag.FlagSet) { "Use :8443 for HTTPS or :8080 for HTTP, or leave as 0 to disable the metrics service.") fs.BoolVar(&o.SecureMetrics, "metrics-secure", true, "If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.") + fs.StringVar(&o.MetricsCertPath, "metrics-cert-path", "", + "The directory that contains the metrics server certificate.") + fs.StringVar(&o.MetricsCertName, "metrics-cert-name", "tls.crt", "The name of the metrics server certificate file.") + fs.StringVar(&o.MetricsCertKey, "metrics-cert-key", "tls.key", "The name of the metrics server key file.") fs.BoolVar(&o.EnableHTTP2, "enable-http2", false, - "If set, HTTP/2 will be enabled for the metrics and webhook servers") + "If set, HTTP/2 will be enabled for the metrics servers") fs.StringVar(&o.ProbeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") fs.BoolVar(&o.EnableLeaderElection, "leader-elect", false, "Enable leader election for controller manager. "+ @@ -187,7 +196,7 @@ func Run(ctx context.Context, opts Options) error { if !opts.EnableHTTP2 { tlsOpts = append(tlsOpts, disableHTTP2) } - // Metrics endpoint is enabled in 'config/default/kustomization.yaml'. The Metrics options configure the server. + // Metrics endpoint is enabled in 'config/volumepoollet-broker/default/kustomization.yaml'. The Metrics options configure the server. // More info: // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.19.1/pkg/metrics/server // - https://book.kubebuilder.io/reference/metrics.html @@ -202,9 +211,37 @@ func Run(ctx context.Context, opts Options) error { // can access the metrics endpoint. The RBAC are configured in 'config/volumepoollet-broker/broker-rbac/kustomization.yaml'. More info: // https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.19.1/pkg/metrics/filters#WithAuthenticationAndAuthorization metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization - // TODO(user): If CertDir, CertName, and KeyName are not specified, controller-runtime will automatically - // generate self-signed certificates for the metrics server. While convenient for development and testing, - // this setup is not recommended for production. + } + + // If the certificate is not specified, controller-runtime will automatically + // generate self-signed certificates for the metrics server. While convenient for development and testing, + // this setup is not recommended for production. + // + // TODO(user): If you enable certManager, uncomment the following lines: + // - [METRICS-WITH-CERTS] at config/volumepoollet-broker/default/kustomization.yaml to generate and use certificates + // managed by cert-manager for the metrics server. + // - [PROMETHEUS-WITH-CERTS] at config/volumepoollet-broker/prometheus/kustomization.yaml for TLS certification. + + // Create watchers for metrics certificates + var metricsCertWatcher *certwatcher.CertWatcher + + if len(opts.MetricsCertPath) > 0 { + setupLog.Info("Initializing metrics certificate watcher using provided certificates", + "metrics-cert-path", opts.MetricsCertPath, "metrics-cert-name", opts.MetricsCertName, "metrics-cert-key", opts.MetricsCertKey) + + var err error + metricsCertWatcher, err = certwatcher.New( + filepath.Join(opts.MetricsCertPath, opts.MetricsCertName), + filepath.Join(opts.MetricsCertPath, opts.MetricsCertKey), + ) + if err != nil { + setupLog.Error(err, "to initialize metrics certificate watcher", "error", err) + os.Exit(1) + } + + metricsServerOptions.TLSOpts = append(metricsServerOptions.TLSOpts, func(config *tls.Config) { + config.GetCertificate = metricsCertWatcher.GetCertificate + }) } mgr, err := ctrl.NewManager(cfg, ctrl.Options{