Skip to content

Commit 59dd9e5

Browse files
fix: expose metrics containerPort to support K8s 1.33 endpoint registration
Kubernetes 1.33 introduced stricter endpoint publishing behavior that requires container ports to be explicitly declared in the pod spec for services to register endpoints. This patch updates `manager_metrics_patch.yaml` to: - add `--metrics-bind-address=:8443` to container args - declare `containerPort: 8443` explicitly Without the port declaration, the metrics service has no endpoints and connections (e.g., from a curl pod) fail with `connection refused`. This ensures metrics access continues to work under network policies in K8s 1.33+.
1 parent b9c3ab6 commit 59dd9e5

File tree

14 files changed

+161
-49
lines changed

14 files changed

+161
-49
lines changed

docs/book/src/cronjob-tutorial/testdata/project/config/default/manager_metrics_patch.yaml

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,3 +2,12 @@
22
- op: add
33
path: /spec/template/spec/containers/0/args/0
44
value: --metrics-bind-address=:8443
5+
6+
# Add the port configuration for the metrics server
7+
# Required for endpoint registration in Kubernetes >= 1.33
8+
- op: add
9+
path: /spec/template/spec/containers/0/ports/-
10+
value:
11+
containerPort: 8443
12+
name: https
13+
protocol: TCP

docs/book/src/cronjob-tutorial/testdata/project/dist/install.yaml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4176,6 +4176,9 @@ spec:
41764176
periodSeconds: 20
41774177
name: manager
41784178
ports:
4179+
- containerPort: 8443
4180+
name: https
4181+
protocol: TCP
41794182
- containerPort: 9443
41804183
name: webhook-server
41814184
protocol: TCP

docs/book/src/getting-started/testdata/project/config/default/manager_metrics_patch.yaml

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,3 +2,12 @@
22
- op: add
33
path: /spec/template/spec/containers/0/args/0
44
value: --metrics-bind-address=:8443
5+
6+
# Add the port configuration for the metrics server
7+
# Required for endpoint registration in Kubernetes >= 1.33
8+
- op: add
9+
path: /spec/template/spec/containers/0/ports/-
10+
value:
11+
containerPort: 8443
12+
name: https
13+
protocol: TCP

docs/book/src/getting-started/testdata/project/dist/install.yaml

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -434,7 +434,10 @@ spec:
434434
initialDelaySeconds: 15
435435
periodSeconds: 20
436436
name: manager
437-
ports: []
437+
ports:
438+
- containerPort: 8443
439+
name: https
440+
protocol: TCP
438441
readinessProbe:
439442
httpGet:
440443
path: /readyz

docs/book/src/multiversion-tutorial/testdata/project/config/default/manager_metrics_patch.yaml

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,3 +2,12 @@
22
- op: add
33
path: /spec/template/spec/containers/0/args/0
44
value: --metrics-bind-address=:8443
5+
6+
# Add the port configuration for the metrics server
7+
# Required for endpoint registration in Kubernetes >= 1.33
8+
- op: add
9+
path: /spec/template/spec/containers/0/ports/-
10+
value:
11+
containerPort: 8443
12+
name: https
13+
protocol: TCP

docs/book/src/multiversion-tutorial/testdata/project/dist/install.yaml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8028,6 +8028,9 @@ spec:
80288028
periodSeconds: 20
80298029
name: manager
80308030
ports:
8031+
- containerPort: 8443
8032+
name: https
8033+
protocol: TCP
80318034
- containerPort: 9443
80328035
name: webhook-server
80338036
protocol: TCP

pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/manager_metrics_patch.go

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,4 +46,13 @@ const kustomizeMetricsPatchTemplate = `# This patch adds the args to allow expos
4646
- op: add
4747
path: /spec/template/spec/containers/0/args/0
4848
value: --metrics-bind-address=:8443
49+
50+
# Add the port configuration for the metrics server
51+
# Required for endpoint registration in Kubernetes >= 1.33
52+
- op: add
53+
path: /spec/template/spec/containers/0/ports/-
54+
value:
55+
containerPort: 8443
56+
name: https
57+
protocol: TCP
4958
`

test/e2e/v4/plugin_cluster_test.go

Lines changed: 79 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -91,25 +91,19 @@ var _ = Describe("kubebuilder", func() {
9191
GenerateV4WithoutMetrics(kbc)
9292
Run(kbc, true, false, false, false, false)
9393
})
94-
// FIXME: This test is currently disabled because it requires to be fixed:
95-
// https://github.com/kubernetes-sigs/kubebuilder/issues/4853
96-
// It is not working for k8s 1.33
97-
// It("should generate a runnable project with metrics protected by network policies", func() {
98-
// GenerateV4WithNetworkPoliciesWithoutWebhooks(kbc)
99-
// Run(kbc, false, false, false, true, true)
100-
// })
94+
It("should generate a runnable project with metrics protected by network policies", func() {
95+
GenerateV4WithNetworkPoliciesWithoutWebhooks(kbc)
96+
Run(kbc, false, false, false, true, true)
97+
})
10198
It("should generate a runnable project with webhooks and metrics protected by network policies", func() {
10299
GenerateV4WithNetworkPolicies(kbc)
103100
Run(kbc, true, false, false, true, true)
104101
})
105-
// FIXME: This test is currently disabled because it requires to be fixed:
106-
// https://github.com/kubernetes-sigs/kubebuilder/issues/4853
107-
// It is not working for k8s 1.33
108-
// It("should generate a runnable project with the manager running "+
109-
// "as restricted and without webhooks", func() {
110-
// GenerateV4WithoutWebhooks(kbc)
111-
// Run(kbc, false, false, false, true, false)
112-
// })
102+
It("should generate a runnable project with the manager running "+
103+
"as restricted and without webhooks", func() {
104+
GenerateV4WithoutWebhooks(kbc)
105+
Run(kbc, false, false, false, true, false)
106+
})
113107
})
114108
})
115109

@@ -151,11 +145,11 @@ func Run(kbc *utils.TestContext, hasWebhook, isToUseInstaller, isToUseHelmChart,
151145
Expect(err).NotTo(HaveOccurred())
152146
}
153147

154-
if isToUseInstaller && !isToUseHelmChart {
155-
By("building the installer")
156-
err = kbc.Make("build-installer", "IMG="+kbc.ImageName)
157-
Expect(err).NotTo(HaveOccurred())
148+
By("building the installer")
149+
err = kbc.Make("build-installer", "IMG="+kbc.ImageName)
150+
Expect(err).NotTo(HaveOccurred())
158151

152+
if isToUseInstaller && !isToUseHelmChart {
159153
By("deploying the controller-manager with the installer")
160154
_, err = kbc.Kubectl.Apply(true, "-f", "dist/install.yaml")
161155
Expect(err).NotTo(HaveOccurred())
@@ -492,10 +486,6 @@ func getMetricsOutput(kbc *utils.TestContext) string {
492486
Expect(err).NotTo(HaveOccurred(), "Failed to check clusterrolebinding existence")
493487
}
494488

495-
token, err := serviceAccountToken(kbc)
496-
Expect(err).NotTo(HaveOccurred())
497-
Expect(token).NotTo(BeEmpty())
498-
499489
var metricsOutput string
500490
By("validating that the controller-manager service is available")
501491
_, err = kbc.Kubectl.Get(
@@ -518,40 +508,80 @@ func getMetricsOutput(kbc *utils.TestContext) string {
518508
Eventually(checkServiceEndpoint, 2*time.Minute, time.Second).Should(Succeed(),
519509
"Service endpoint should be ready")
520510

511+
By("waiting briefly to ensure controller is listening on port 8443")
512+
time.Sleep(15 * time.Second)
513+
514+
token, err := serviceAccountToken(kbc)
515+
Expect(err).NotTo(HaveOccurred())
516+
Expect(token).NotTo(BeEmpty())
517+
podName := fmt.Sprintf("curl-%s", kbc.TestSuffix)
521518
By("creating a curl pod to access the metrics endpoint")
522-
cmdOpts := cmdOptsToCreateCurlPod(kbc, token)
519+
cmdOpts := cmdOptsToCreateCurlPod(kbc, token, podName)
523520
_, err = kbc.Kubectl.CommandInNamespace(cmdOpts...)
524521
Expect(err).NotTo(HaveOccurred())
525522

526523
By("validating that the curl pod is running as expected")
527524
verifyCurlUp := func(g Gomega) {
528-
var status string
529-
status, err = kbc.Kubectl.Get(
530-
true,
531-
"pods", "curl", "-o", "jsonpath={.status.phase}")
525+
maxRetries := 3
526+
var status, logs string
527+
528+
for i := 0; i < maxRetries; i++ {
529+
status, err = kbc.Kubectl.Get(
530+
true,
531+
"pods", podName, "-o", "jsonpath={.status.phase}")
532+
g.Expect(err).NotTo(HaveOccurred())
533+
534+
if status == "Succeeded" {
535+
return
536+
}
537+
538+
logs, _ = kbc.Kubectl.Logs(podName)
539+
540+
if status == "Failed" ||
541+
strings.Contains(logs, "Failed to connect") ||
542+
strings.Contains(logs, "Connection refused") {
543+
By("Outputting curl pod logs for debugging")
544+
_, _ = fmt.Fprintln(GinkgoWriter, logs)
545+
546+
By("Outputting manager pod logs for debugging")
547+
controllerPodName := getControllerName(kbc)
548+
managerLogs, _ := kbc.Kubectl.Logs(controllerPodName)
549+
_, _ = fmt.Fprintln(GinkgoWriter, managerLogs)
550+
551+
By("Describing all resources for debugging")
552+
out, errDescribe := kbc.Kubectl.CommandInNamespace("describe", "all")
553+
Expect(errDescribe).NotTo(HaveOccurred())
554+
_, _ = fmt.Fprintln(GinkgoWriter, out)
555+
556+
By(fmt.Sprintf("curl pod failed with status %s. Retrying (%d/%d)...", status, i+1, maxRetries))
557+
_, _ = kbc.Kubectl.Delete(true, "pod", podName, "--ignore-not-found",
558+
"--grace-period=0", "--force")
559+
time.Sleep(3 * time.Second)
560+
561+
cmdOpts = cmdOptsToCreateCurlPod(kbc, token, podName)
562+
_, err = kbc.Kubectl.CommandInNamespace(cmdOpts...)
563+
g.Expect(err).NotTo(HaveOccurred())
564+
565+
time.Sleep(5 * time.Second)
566+
} else {
567+
By(fmt.Sprintf("curl pod in %s state without known failure, waiting...", status))
568+
time.Sleep(5 * time.Second)
569+
}
570+
}
571+
status, err = kbc.Kubectl.Get(true, "pods", podName, "-o", "jsonpath={.status.phase}")
532572
g.Expect(err).NotTo(HaveOccurred())
533-
g.Expect(status).To(Equal("Succeeded"), fmt.Sprintf("curl pod in %s status", status))
573+
g.Expect(status).To(Equal("Succeeded"), fmt.Sprintf("curl pod in %s state after all retries", status))
534574
}
535-
Eventually(verifyCurlUp, 240*time.Second, time.Second).Should(Succeed())
536-
537-
By("validating that the correct ServiceAccount is being used")
538-
saName := kbc.Kubectl.ServiceAccount
539-
currentSAOutput, err := kbc.Kubectl.Get(
540-
true,
541-
"serviceaccount", saName,
542-
"-o", "jsonpath={.metadata.name}",
543-
)
544-
Expect(err).NotTo(HaveOccurred(), "Failed to fetch the service account")
545-
Expect(currentSAOutput).To(Equal(saName), "The ServiceAccount in use does not match the expected one")
575+
Eventually(verifyCurlUp, 2*time.Minute, time.Second).Should(Succeed())
546576

547577
By("validating that the metrics endpoint is serving as expected")
548578
getCurlLogs := func(g Gomega) {
549-
metricsOutput, err = kbc.Kubectl.Logs("curl")
579+
metricsOutput, err = kbc.Kubectl.Logs(podName)
550580
g.Expect(err).NotTo(HaveOccurred())
551581
g.Expect(metricsOutput).Should(ContainSubstring("< HTTP/1.1 200 OK"))
552582
}
553583
Eventually(getCurlLogs, 10*time.Second, time.Second).Should(Succeed())
554-
removeCurlPod(kbc)
584+
removeCurlPod(kbc, podName)
555585
return metricsOutput
556586
}
557587

@@ -565,9 +595,10 @@ func metricsShouldBeUnavailable(kbc *utils.TestContext) {
565595
token, err := serviceAccountToken(kbc)
566596
Expect(err).NotTo(HaveOccurred())
567597
Expect(token).NotTo(BeEmpty())
598+
podName := fmt.Sprintf("curl-%s", kbc.TestSuffix)
568599

569600
By("creating a curl pod to access the metrics endpoint")
570-
cmdOpts := cmdOptsToCreateCurlPod(kbc, token)
601+
cmdOpts := cmdOptsToCreateCurlPod(kbc, token, podName)
571602
_, err = kbc.Kubectl.CommandInNamespace(cmdOpts...)
572603
Expect(err).NotTo(HaveOccurred())
573604

@@ -589,13 +620,13 @@ func metricsShouldBeUnavailable(kbc *utils.TestContext) {
589620
g.Expect(metricsOutput).Should(ContainSubstring("Could not resolve host"))
590621
}
591622
Eventually(getCurlLogs, 10*time.Second, time.Second).Should(Succeed())
592-
removeCurlPod(kbc)
623+
removeCurlPod(kbc, podName)
593624
}
594625

595-
func cmdOptsToCreateCurlPod(kbc *utils.TestContext, token string) []string {
626+
func cmdOptsToCreateCurlPod(kbc *utils.TestContext, token, podName string) []string {
596627
//nolint:lll
597628
cmdOpts := []string{
598-
"run", "curl",
629+
"run", podName,
599630
"--restart=Never",
600631
"--namespace", kbc.Kubectl.Namespace,
601632
"--image=curlimages/curl:latest",
@@ -627,9 +658,9 @@ func cmdOptsToCreateCurlPod(kbc *utils.TestContext, token string) []string {
627658
return cmdOpts
628659
}
629660

630-
func removeCurlPod(kbc *utils.TestContext) {
661+
func removeCurlPod(kbc *utils.TestContext, podName string) {
631662
By("cleaning up the curl pod")
632-
_, err := kbc.Kubectl.Delete(true, "pods/curl", "--grace-period=0", "--force")
663+
_, err := kbc.Kubectl.Delete(true, fmt.Sprintf("pods/%s", podName), "--grace-period=0", "--force")
633664
Expect(err).NotTo(HaveOccurred())
634665
}
635666

testdata/project-v4-multigroup/config/default/manager_metrics_patch.yaml

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,3 +2,12 @@
22
- op: add
33
path: /spec/template/spec/containers/0/args/0
44
value: --metrics-bind-address=:8443
5+
6+
# Add the port configuration for the metrics server
7+
# Required for endpoint registration in Kubernetes >= 1.33
8+
- op: add
9+
path: /spec/template/spec/containers/0/ports/-
10+
value:
11+
containerPort: 8443
12+
name: https
13+
protocol: TCP

testdata/project-v4-multigroup/dist/install.yaml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2137,6 +2137,9 @@ spec:
21372137
periodSeconds: 20
21382138
name: manager
21392139
ports:
2140+
- containerPort: 8443
2141+
name: https
2142+
protocol: TCP
21402143
- containerPort: 9443
21412144
name: webhook-server
21422145
protocol: TCP

testdata/project-v4-with-plugins/config/default/manager_metrics_patch.yaml

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,3 +2,12 @@
22
- op: add
33
path: /spec/template/spec/containers/0/args/0
44
value: --metrics-bind-address=:8443
5+
6+
# Add the port configuration for the metrics server
7+
# Required for endpoint registration in Kubernetes >= 1.33
8+
- op: add
9+
path: /spec/template/spec/containers/0/ports/-
10+
value:
11+
containerPort: 8443
12+
name: https
13+
protocol: TCP

testdata/project-v4-with-plugins/dist/install.yaml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -830,6 +830,9 @@ spec:
830830
periodSeconds: 20
831831
name: manager
832832
ports:
833+
- containerPort: 8443
834+
name: https
835+
protocol: TCP
833836
- containerPort: 9443
834837
name: webhook-server
835838
protocol: TCP

testdata/project-v4/config/default/manager_metrics_patch.yaml

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,3 +2,12 @@
22
- op: add
33
path: /spec/template/spec/containers/0/args/0
44
value: --metrics-bind-address=:8443
5+
6+
# Add the port configuration for the metrics server
7+
# Required for endpoint registration in Kubernetes >= 1.33
8+
- op: add
9+
path: /spec/template/spec/containers/0/ports/-
10+
value:
11+
containerPort: 8443
12+
name: https
13+
protocol: TCP

testdata/project-v4/dist/install.yaml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -695,6 +695,9 @@ spec:
695695
periodSeconds: 20
696696
name: manager
697697
ports:
698+
- containerPort: 8443
699+
name: https
700+
protocol: TCP
698701
- containerPort: 9443
699702
name: webhook-server
700703
protocol: TCP

0 commit comments

Comments
 (0)