Skip to content

Commit f5f1734

Browse files
sujeet01sujeet01
committed
Enhance cert-manager integration for metrics endpoints
1 parent 2e4d763 commit f5f1734

File tree

24 files changed

+892
-301
lines changed

24 files changed

+892
-301
lines changed

‎cmd/ironcore-controller-manager/main.go

Lines changed: 39 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import (
88
"flag"
99
"fmt"
1010
"os"
11+
"path/filepath"
1112
"time"
1213

1314
corev1alpha1 "github.com/ironcore-dev/ironcore/api/core/v1alpha1"
@@ -29,6 +30,7 @@ import (
2930
quotaevaluatorironcore "github.com/ironcore-dev/ironcore/internal/quota/evaluator/ironcore"
3031
"github.com/ironcore-dev/ironcore/utils/quota"
3132
"k8s.io/utils/lru"
33+
"sigs.k8s.io/controller-runtime/pkg/certwatcher"
3234
"sigs.k8s.io/controller-runtime/pkg/metrics/filters"
3335
metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
3436

@@ -103,6 +105,7 @@ func init() {
103105
func main() {
104106
var metricsAddr string
105107
var secureMetrics bool
108+
var metricsCertPath, metricsCertName, metricsCertKey string
106109
var enableHTTP2 bool
107110
var enableLeaderElection bool
108111
var probeAddr string
@@ -115,8 +118,12 @@ func main() {
115118
"Use :8443 for HTTPS or :8080 for HTTP, or leave as 0 to disable the metrics service.")
116119
flag.BoolVar(&secureMetrics, "metrics-secure", true,
117120
"If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.")
121+
flag.StringVar(&metricsCertPath, "metrics-cert-path", "",
122+
"The directory that contains the metrics server certificate.")
123+
flag.StringVar(&metricsCertName, "metrics-cert-name", "tls.crt", "The name of the metrics server certificate file.")
124+
flag.StringVar(&metricsCertKey, "metrics-cert-key", "tls.key", "The name of the metrics server key file.")
118125
flag.BoolVar(&enableHTTP2, "enable-http2", false,
119-
"If set, HTTP/2 will be enabled for the metrics and webhook servers")
126+
"If set, HTTP/2 will be enabled for the metrics servers")
120127
flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
121128
flag.BoolVar(&enableLeaderElection, "leader-elect", false,
122129
"Enable leader election for controller manager. "+
@@ -191,7 +198,7 @@ func main() {
191198
tlsOpts = append(tlsOpts, disableHTTP2)
192199
}
193200

194-
// Metrics endpoint is enabled in 'config/default/kustomization.yaml'. The Metrics options configure the server.
201+
// Metrics endpoint is enabled in 'config/controller/default/kustomization.yaml'. The Metrics options configure the server.
195202
// More info:
196203
// - https://pkg.go.dev/sigs.k8s.io/[email protected]/pkg/metrics/server
197204
// - https://book.kubebuilder.io/reference/metrics.html
@@ -207,10 +214,37 @@ func main() {
207214
// can access the metrics endpoint. The RBAC are configured in 'config/controller/rbac/kustomization.yaml'. More info:
208215
// https://pkg.go.dev/sigs.k8s.io/[email protected]/pkg/metrics/filters#WithAuthenticationAndAuthorization
209216
metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization
217+
}
218+
219+
// If the certificate is not specified, controller-runtime will automatically
220+
// generate self-signed certificates for the metrics server. While convenient for development and testing,
221+
// this setup is not recommended for production.
222+
//
223+
// TODO(user): If you enable certManager, uncomment the following lines:
224+
// - [METRICS-WITH-CERTS] at config/controller/default/kustomization.yaml to generate and use certificates
225+
// managed by cert-manager for the metrics server.
226+
// - [PROMETHEUS-WITH-CERTS] at config/controller/prometheus/kustomization.yaml for TLS certification.
227+
228+
// Create watchers for metrics certificates
229+
var metricsCertWatcher *certwatcher.CertWatcher
230+
231+
if len(metricsCertPath) > 0 {
232+
setupLog.Info("Initializing metrics certificate watcher using provided certificates",
233+
"metrics-cert-path", metricsCertPath, "metrics-cert-name", metricsCertName, "metrics-cert-key", metricsCertKey)
234+
235+
var err error
236+
metricsCertWatcher, err = certwatcher.New(
237+
filepath.Join(metricsCertPath, metricsCertName),
238+
filepath.Join(metricsCertPath, metricsCertKey),
239+
)
240+
if err != nil {
241+
setupLog.Error(err, "to initialize metrics certificate watcher", "error", err)
242+
os.Exit(1)
243+
}
210244

211-
// TODO(user): If CertDir, CertName, and KeyName are not specified, controller-runtime will automatically
212-
// generate self-signed certificates for the metrics server. While convenient for development and testing,
213-
// this setup is not recommended for production.
245+
metricsServerOptions.TLSOpts = append(metricsServerOptions.TLSOpts, func(config *tls.Config) {
246+
config.GetCertificate = metricsCertWatcher.GetCertificate
247+
})
214248
}
215249

216250
mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{

‎config/bucketpoollet-broker/default/cert_metrics_manager_patch.yaml

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
# This patch adds the args, volumes, and ports to allow the manager to use the metrics-server certs.
2+
apiVersion: apps/v1
3+
kind: Deployment
4+
metadata:
5+
name: controller-manager
6+
namespace: system
7+
spec:
8+
template:
9+
spec:
10+
containers:
11+
- name: manager
12+
volumeMounts:
13+
- mountPath: /tmp/k8s-metrics-server/metrics-certs
14+
name: metrics-certs
15+
readOnly: true
16+
args:
17+
- "--metrics-cert-path=/tmp/k8s-metrics-server/metrics-certs"
18+
volumes:
19+
- name: metrics-certs
20+
secret:
21+
secretName: metrics-server-cert
22+
optional: false
23+
items:
24+
- key: ca.crt
25+
path: ca.crt
26+
- key: tls.crt
27+
path: tls.crt
28+
- key: tls.key
29+
path: tls.key

‎config/bucketpoollet-broker/default/kustomization.yaml

Lines changed: 138 additions & 92 deletions
Original file line numberDiff line numberDiff line change
@@ -13,101 +13,147 @@ namePrefix: bucketpoollet-
1313
# someName: someValue
1414

1515
resources:
16-
- ../broker-rbac
17-
- ../manager
18-
# - ../webhook
19-
# - ../certmanager
16+
- ../broker-rbac
17+
- ../manager
18+
# - ../webhook
19+
# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. 'WEBHOOK' components are required.
20+
# - ../certmanager
2021
# [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'.
21-
#- ../prometheus
22+
# - ../prometheus
2223

2324
patchesStrategicMerge:
24-
- manager_metrics_patch.yaml
25+
# [METRICS] The following patch will enable the metrics endpoint using HTTPS and the port :8443.
26+
# More info: https://book.kubebuilder.io/reference/metrics
27+
- manager_metrics_patch.yaml
28+
29+
# Uncomment the patches line if you enable Metrics and CertManager
30+
# [METRICS-WITH-CERTS] To enable metrics protected with certManager, uncomment the following line.
31+
# This patch will protect the metrics with certManager self-signed certs.
32+
# - cert_metrics_manager_patch.yaml
2533

26-
# Mount the controller config file for loading manager configurations
27-
# through a ComponentConfig type
28-
#- manager_config_patch.yaml
34+
# Mount the controller config file for loading manager configurations
35+
# through a ComponentConfig type
36+
#- manager_config_patch.yaml
2937

30-
#- manager_webhook_patch.yaml
31-
#- webhookcainjection_patch.yaml
38+
#- manager_webhook_patch.yaml
39+
#- webhookcainjection_patch.yaml
3240

33-
#replacements:
34-
# - source:
35-
# kind: Certificate
36-
# group: cert-manager.io
37-
# version: v1
38-
# name: serving-cert
39-
# fieldPath: .metadata.namespace
40-
# targets:
41-
# - select:
42-
# kind: ValidatingWebhookConfiguration
43-
# fieldPaths:
44-
# - .metadata.annotations.[cert-manager.io/inject-ca-from]
45-
# options:
46-
# delimiter: '/'
47-
# index: 0
48-
# create: true
49-
# - select:
50-
# kind: MutatingWebhookConfiguration
51-
# fieldPaths:
52-
# - .metadata.annotations.[cert-manager.io/inject-ca-from]
53-
# options:
54-
# delimiter: '/'
55-
# index: 0
56-
# create: true
57-
# - source:
58-
# kind: Certificate
59-
# group: cert-manager.io
60-
# version: v1
61-
# name: serving-cert
62-
# fieldPath: .metadata.name
63-
# targets:
64-
# - select:
65-
# kind: ValidatingWebhookConfiguration
66-
# fieldPaths:
67-
# - .metadata.annotations.[cert-manager.io/inject-ca-from]
68-
# options:
69-
# delimiter: '/'
70-
# index: 1
71-
# create: true
72-
# - select:
73-
# kind: MutatingWebhookConfiguration
74-
# fieldPaths:
75-
# - .metadata.annotations.[cert-manager.io/inject-ca-from]
76-
# options:
77-
# delimiter: '/'
78-
# index: 1
79-
# create: true
80-
# - source:
81-
# kind: Service
82-
# version: v1
83-
# name: webhook-service
84-
# fieldPath: .metadata.name
85-
# targets:
86-
# - select:
87-
# kind: Certificate
88-
# group: cert-manager.io
89-
# version: v1
90-
# fieldPaths:
91-
# - .spec.dnsNames.0
92-
# - .spec.dnsNames.1
93-
# options:
94-
# delimiter: '.'
95-
# index: 0
96-
# create: true
97-
# - source:
98-
# kind: Service
99-
# version: v1
100-
# name: webhook-service
101-
# fieldPath: .metadata.namespace
102-
# targets:
103-
# - select:
104-
# kind: Certificate
105-
# group: cert-manager.io
106-
# version: v1
107-
# fieldPaths:
108-
# - .spec.dnsNames.0
109-
# - .spec.dnsNames.1
110-
# options:
111-
# delimiter: '.'
112-
# index: 1
113-
# create: true
41+
# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix.
42+
# Uncomment the following replacements to add the cert-manager CA injection annotations
43+
# replacements:
44+
# - source: # Uncomment the following block to enable certificates for metrics
45+
# kind: Service
46+
# version: v1
47+
# name: controller-manager-metrics-service
48+
# fieldPath: metadata.name
49+
# targets:
50+
# - select:
51+
# kind: Certificate
52+
# group: cert-manager.io
53+
# version: v1
54+
# name: metrics-certs
55+
# fieldPaths:
56+
# - spec.dnsNames.0
57+
# - spec.dnsNames.1
58+
# options:
59+
# delimiter: '.'
60+
# index: 0
61+
# create: true
62+
# - source:
63+
# kind: Service
64+
# version: v1
65+
# name: controller-manager-metrics-service
66+
# fieldPath: metadata.namespace
67+
# targets:
68+
# - select:
69+
# kind: Certificate
70+
# group: cert-manager.io
71+
# version: v1
72+
# name: metrics-certs
73+
# fieldPaths:
74+
# - spec.dnsNames.0
75+
# - spec.dnsNames.1
76+
# options:
77+
# delimiter: '.'
78+
# index: 1
79+
# create: true
80+
# - source:
81+
# kind: Certificate
82+
# group: cert-manager.io
83+
# version: v1
84+
# name: serving-cert
85+
# fieldPath: .metadata.namespace
86+
# targets:
87+
# - select:
88+
# kind: ValidatingWebhookConfiguration
89+
# fieldPaths:
90+
# - .metadata.annotations.[cert-manager.io/inject-ca-from]
91+
# options:
92+
# delimiter: '/'
93+
# index: 0
94+
# create: true
95+
# - select:
96+
# kind: MutatingWebhookConfiguration
97+
# fieldPaths:
98+
# - .metadata.annotations.[cert-manager.io/inject-ca-from]
99+
# options:
100+
# delimiter: '/'
101+
# index: 0
102+
# create: true
103+
# - source:
104+
# kind: Certificate
105+
# group: cert-manager.io
106+
# version: v1
107+
# name: serving-cert
108+
# fieldPath: .metadata.name
109+
# targets:
110+
# - select:
111+
# kind: ValidatingWebhookConfiguration
112+
# fieldPaths:
113+
# - .metadata.annotations.[cert-manager.io/inject-ca-from]
114+
# options:
115+
# delimiter: '/'
116+
# index: 1
117+
# create: true
118+
# - select:
119+
# kind: MutatingWebhookConfiguration
120+
# fieldPaths:
121+
# - .metadata.annotations.[cert-manager.io/inject-ca-from]
122+
# options:
123+
# delimiter: '/'
124+
# index: 1
125+
# create: true
126+
# - source:
127+
# kind: Service
128+
# version: v1
129+
# name: webhook-service
130+
# fieldPath: .metadata.name
131+
# targets:
132+
# - select:
133+
# kind: Certificate
134+
# group: cert-manager.io
135+
# version: v1
136+
# fieldPaths:
137+
# - .spec.dnsNames.0
138+
# - .spec.dnsNames.1
139+
# options:
140+
# delimiter: '.'
141+
# index: 0
142+
# create: true
143+
# - source:
144+
# kind: Service
145+
# version: v1
146+
# name: webhook-service
147+
# fieldPath: .metadata.namespace
148+
# targets:
149+
# - select:
150+
# kind: Certificate
151+
# group: cert-manager.io
152+
# version: v1
153+
# fieldPaths:
154+
# - .spec.dnsNames.0
155+
# - .spec.dnsNames.1
156+
# options:
157+
# delimiter: '.'
158+
# index: 1
159+
# create: true

‎config/bucketpoollet-broker/prometheus/kustomization.yaml

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,11 @@
11
resources:
22
- monitor.yaml
3+
4+
# [PROMETHEUS-WITH-CERTS] The following patch configures the ServiceMonitor in ../prometheus
5+
# to securely reference certificates created and managed by cert-manager.
6+
# Additionally, ensure that you uncomment the [METRICS-WITH-CERTS] patch under config/bucketpoollet-broker/default/kustomization.yaml
7+
# to mount the "metrics-server-cert" secret in the Manager Deployment.
8+
#patches:
9+
# - path: monitor_tls_patch.yaml
10+
# target:
11+
# kind: ServiceMonitor

‎config/bucketpoollet-broker/prometheus/monitor.yaml

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,10 +10,15 @@ metadata:
1010
spec:
1111
endpoints:
1212
- path: /metrics
13-
port: https
13+
port: https # Ensure this is the name of the port that exposes HTTPS metrics
1414
scheme: https
1515
bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
1616
tlsConfig:
17+
# TODO(user): The option insecureSkipVerify: true is not recommended for production since it disables
18+
# certificate verification, exposing the system to potential man-in-the-middle attacks.
19+
# For production environments, it is recommended to use cert-manager for automatic TLS certificate management.
20+
# To apply this configuration, enable cert-manager and use the patch located at config/bucketpoollet-broker/prometheus/monitor_tls_patch.yaml,
21+
# which securely references the certificate from the 'metrics-server-cert' secret.
1722
insecureSkipVerify: true
1823
selector:
1924
matchLabels:

0 commit comments

Comments
 (0)